Add ldap search for unprovisioned projects

Also, move get_projects_to_setup() functionality into several different methods for clarity.

Author: williamnswanson

project_group_setup.py

@@ -4,13 +4,15 @@
 import re
 import sys
 import json
-#import ldap3
 import getopt
 import urllib.error
 import urllib.request
+from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC
 
 SCRIPT = os.path.basename(__file__)
 ENDPOINT = "https://registry-test.cilogon.org/registry/"
+LDAP_SERVER = "ldaps://ldap.cilogon.org"
+LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
 OSG_CO_ID = 8
 UNIX_CLUSTER_ID = 10
 LDAP_TARGET_ID = 9
@@ -35,6 +37,7 @@
   -c OSG_CO_ID        specify OSG CO ID (default = {OSG_CO_ID})
   -g CLUSTER_ID       specify UNIX Cluster ID (default = {UNIX_CLUSTER_ID})
   -l LDAP_TARGET      specify LDAP Provsion ID (defult = {LDAP_TARGET_ID})
+  -p LDAP authtok     specify LDAP server authtok
   -d passfd           specify open fd to read PASS
   -f passfile         specify path to file to open and read PASS
   -e ENDPOINT         specify REST endpoint
@@ -68,6 +71,7 @@ class Options:
     provision_target = LDAP_TARGET_ID
     outfile = None
     authstr = None
+    ldap_authtok = None
     min_timeout = MINTIMEOUT
     max_timeout = MAXTIMEOUT
     project_gid_startval = PROJECT_GIDS_START
@@ -169,6 +173,16 @@ def get_datalist(data, listname):
     return data[listname] if data else []
 
 
+def get_ldap_groups():
+    ldap_group_osggids = set()
+    server = Server(LDAP_SERVER, get_info=ALL)
+    connection = Connection(server, LDAP_USER, options.ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
+    _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES)
+    for group in response:
+        ldap_group_osggids.add(group["attributes"]["gidNumber"])
+    return ldap_group_osggids
+
+
 def identifier_from_list(id_list, id_type):
     id_type_list = [id["Type"] for id in id_list]
     try:
@@ -216,7 +230,7 @@ def ldap_provision_group(gid):
 
 def parse_options(args):
     try:
-        ops, args = getopt.getopt(args, "u:c:d:f:e:o:t:T:h")
+        ops, args = getopt.getopt(args, "u:c:g:l:p:d:f:e:o:t:T:h")
     except getopt.GetoptError:
         usage()
 
@@ -237,6 +251,8 @@ def parse_options(args):
             options.ucid = int(arg)
         if op == "-l":
             options.provision_target = int(arg)
+        if op == "-p":
+            options.ldap_authtok = arg
         if op == "-d":
             passfd = int(arg)
         if op == "-f":
@@ -267,6 +283,8 @@ def update_highest_osggid(highest_osggid, group):
     # If this group has a osggid, keep a hold of the highest one we've seen so far
     if osggid is not None:
         return max(highest_osggid, int(osggid))
+    else:
+        return highest_osggid
 
 
 def get_comanage_data():
@@ -287,28 +305,60 @@ def get_comanage_data():
     return comanage_data
 
 
-def get_projects_to_setup(project_groups):
-    projects_to_setup = {
-        "Need Identifiers": [],
-        "Need Cluster Groups": [],
-        "Need Provisioning": [],
-    }
-
-    # CO Groups associated with a UNIX Cluster Group
-    clustered_group_ids = get_unix_cluster_groups_ids(options.ucid)
-
+def get_projects_needing_identifiers(project_groups):
+    projects_needing_identifiers = []
     for project in project_groups:
         # If this project doesn't have an osggid already assigned to it...
         if identifier_from_list(project["ID_List"], "osggid") is None:
             # Prep the project to have the proper identifiers added to it
-            projects_to_setup["Need Identifiers"].append(project)
+            projects_needing_identifiers.append(project)
+    return projects_needing_identifiers
 
-        # If this project doesn't have a UNIX Cluster Group associated with it...
-        if not project["Gid"] in clustered_group_ids:
-            # Prep it to have one made for it and to be provisioned in LDAP
-            projects_to_setup["Need Cluster Groups"].append(project)
-            projects_to_setup["Need Provisioning"].append(project)
 
+def get_projects_needing_cluster_groups(project_groups):
+    # CO Groups associated with a UNIX Cluster Group
+    clustered_group_ids = get_unix_cluster_groups_ids(options.ucid)
+    try:
+        # All project Gids
+        project_gids = set(project["Gid"] for project in project_groups)
+        # Project Gids for projects without UNIX cluster groups
+        project_gids_lacking_cluster_groups = project_gids.difference(clustered_group_ids)
+        # All projects needing UNIX cluster groups
+        projects_needing_unix_groups = (
+            project
+            for project in project_groups
+            if project["Gid"] in project_gids_lacking_cluster_groups
+        )
+        return projects_needing_unix_groups
+    except TypeError:
+        return set()
+    
+
+def get_projects_needing_provisioning(project_groups):
+    # project groups provisioned in LDAP
+    ldap_group_osggids = get_ldap_groups()
+    try:
+        # All project osggids
+        project_osggids = set(identifier_from_list(project["ID_List"], "osggid") for project in project_groups)
+        # project osggids not provisioned in ldap
+        project_osggids_to_provision = project_osggids.difference(ldap_group_osggids)
+        # All projects that aren't provisioned in ldap
+        projects_to_provision = (
+            project
+            for project in project_groups
+            if identifier_from_list(project["ID_List"], "osggid") in project_osggids_to_provision
+        )
+        return projects_to_provision
+    except TypeError:
+        return set()
+
+
+def get_projects_to_setup(project_groups):
+    projects_to_setup = {
+        "Need Identifiers": get_projects_needing_identifiers(project_groups),
+        "Need Cluster Groups": get_projects_needing_cluster_groups(project_groups),
+        "Need Provisioning": get_projects_needing_provisioning(project_groups),
+    }
     return projects_to_setup