groups issue with openssh (all versions since at least 3.8), AIX 5.3 and NIS

[Darren Tucker]

On Tue, Mar 14, 2006 at 02:11:41PM +1100, Darren Tucker wrote:
> On Mon, Mar 13, 2006 at 07:47:37PM -0600, Gael Martinez wrote:
> [...]
> > that in average a ssh connection is generating over 12000 nis sessions,
> > scanning basically all the group.byname table a few times and we got a
> > few thousands groups... :(
> [...]
> > Let me know, I will assist as much as possible, this is really a big
> > issue for us, and I'm not able to determine if that issue 
> > can be resolved with a patch to openssh or at the OS level.
> 
> Looking briefly at the URL and the code, it looks like it could be
> resolved by implementing an AIX-specific getgroupslist() based on
> getgrset().  I'll look at at it if you can test patches.

Please try this diff (against OpenSSH 4.3p2).

-- 
Darren Tucker (dtucker at zip.com.au)
GPG key 8FF4FA69 / D9A3 86E9 7EEE AF4B B2D4  37C9 C982 80C7 8FF4 FA69
    Good judgement comes with experience. Unfortunately, the experience
usually comes from bad judgement.
-------------- next part --------------
Only in openssh-4.3p2: autom4te.cache
diff -ru openssh-4.3p2.orig/config.h.in openssh-4.3p2/config.h.in
--- openssh-4.3p2.orig/config.h.in	2006-02-11 11:07:35.000000000 +1100
+++ openssh-4.3p2/config.h.in	2006-03-14 21:54:49.000000000 +1100
@@ -305,6 +305,9 @@
 /* Define to 1 if you have the `getgrouplist' function. */
 #undef HAVE_GETGROUPLIST
 
+/* Define to 1 if you have the `getgrset' function. */
+#undef HAVE_GETGRSET
+
 /* Define to 1 if you have the `getluid' function. */
 #undef HAVE_GETLUID
 
Only in openssh-4.3p2: config.h.in~
diff -ru openssh-4.3p2.orig/configure openssh-4.3p2/configure
--- openssh-4.3p2.orig/configure	2006-02-11 11:07:37.000000000 +1100
+++ openssh-4.3p2/configure	2006-03-14 21:55:13.000000000 +1100
@@ -5019,7 +5019,8 @@
 
 
 
-for ac_func in setauthdb
+
+for ac_func in getgrset setauthdb
 do
 as_ac_var=`echo "ac_cv_func_$ac_func" | $as_tr_sh`
 echo "$as_me:$LINENO: checking for $ac_func" >&5
@@ -27224,9 +27225,9 @@
 exec 5>>config.log
 {
   echo
-  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<BOXI_EOF
+  sed 'h;s/./-/g;s/^.../## /;s/...$/ ##/;p;x;p;x' <<_ASBOX
 ## Running $as_me. ##
-BOXI_EOF
+_ASBOX
 } >&5
 cat >&5 <<_CSEOF
 
diff -ru openssh-4.3p2.orig/configure.ac openssh-4.3p2/configure.ac
--- openssh-4.3p2.orig/configure.ac	2006-02-08 22:11:06.000000000 +1100
+++ openssh-4.3p2/configure.ac	2006-03-14 21:53:53.000000000 +1100
@@ -174,7 +174,7 @@
 		[],
 		[#include <usersec.h>]
 	)
-	AC_CHECK_FUNCS(setauthdb)
+	AC_CHECK_FUNCS(getgrset setauthdb)
 	check_for_aix_broken_getaddrinfo=1
 	AC_DEFINE(BROKEN_REALPATH, 1, [Define if you have a broken realpath.])
 	AC_DEFINE(SETEUID_BREAKS_SETUID, 1,
diff -ru openssh-4.3p2.orig/openbsd-compat/port-aix.c openssh-4.3p2/openbsd-compat/port-aix.c
--- openssh-4.3p2.orig/openbsd-compat/port-aix.c	2005-05-29 10:54:28.000000000 +1000
+++ openssh-4.3p2/openbsd-compat/port-aix.c	2006-03-14 21:53:53.000000000 +1100
@@ -1,7 +1,7 @@
 /*
  *
  * Copyright (c) 2001 Gert Doering.  All rights reserved.
- * Copyright (c) 2003,2004,2005 Darren Tucker.  All rights reserved.
+ * Copyright (c) 2003,2004,2005,2006 Darren Tucker.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -372,4 +372,47 @@
 }
 # endif /* AIX_GETNAMEINFO_HACK */
 
+# if defined(USE_GETGRSET)
+#  include <stdlib.h>
+int
+getgrouplist(const char *user, gid_t pgid, gid_t *groups, int *grpcnt)
+{
+	char *cp, *grplist, *grp;
+	gid_t gid;
+	int ret = 0, ngroups = 0, maxgroups;
+	long l;
+
+	maxgroups = *grpcnt;
+
+	if ((cp = grplist = getgrset(user)) == NULL)
+		return -1;
+
+	/* handle zero-length case */
+	if (maxgroups <= 0) {
+		*grpcnt = 0;
+		return -1;
+	}
+
+	/* copy primary group */
+	groups[ngroups++] = pgid;
+
+	/* copy each entry from getgrset into group list */
+	while ((grp = strsep(&grplist, ",")) != NULL) {
+		l = strtol(grp, NULL, 10);
+		if (ngroups >= maxgroups || l == LONG_MIN || l == LONG_MAX) {
+			ret = -1;
+			goto out;
+		}
+		gid = (gid_t)l;
+		if (gid == pgid)
+			continue;	/* we have already added primary gid */
+		groups[ngroups++] = gid;
+	}
+out:
+	free(cp);
+	*grpcnt = ngroups;
+	return ret;
+}
+# endif	/* USE_GETGRSET */
+
 #endif /* _AIX */
diff -ru openssh-4.3p2.orig/openbsd-compat/port-aix.h openssh-4.3p2/openbsd-compat/port-aix.h
--- openssh-4.3p2.orig/openbsd-compat/port-aix.h	2005-05-28 20:28:40.000000000 +1000
+++ openssh-4.3p2/openbsd-compat/port-aix.h	2006-03-14 21:53:53.000000000 +1100
@@ -3,7 +3,7 @@
 /*
  *
  * Copyright (c) 2001 Gert Doering.  All rights reserved.
- * Copyright (c) 2004, 2005 Darren Tucker.  All rights reserved.
+ * Copyright (c) 2004,2005,2006 Darren Tucker.  All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -115,4 +115,11 @@
 # define getnameinfo(a,b,c,d,e,f,g) (sshaix_getnameinfo(a,b,c,d,e,f,g))
 #endif
 
+/* We use getgrset in preference to multiple getgrent calls for efficiency */
+#if !defined(HAVE_GETGRENT) && defined(HAVE_GETGRSET)
+# define HAVE_GETGRENT
+# define USE_GETGRSET
+int getgrouplist(const char *, gid_t, gid_t *, int *);
+#endif
+
 #endif /* _AIX */