Merge branch 'master' into INF-2987.mass-person-create-update-script

Author: williamnswanson

Dockerfile

@@ -1,4 +1,4 @@
-FROM library/python:3.7-alpine
+FROM library/python:3.9-alpine
 
 LABEL maintainer OSG Software <help@opensciencegrid.org>
 

comanage_utils.py

@@ -4,34 +4,63 @@
 import re
 import json
 import time
+import configparser
 import urllib.error
 import urllib.request
-from ldap3 import Server, Connection, ALL, ALL_ATTRIBUTES, SAFE_SYNC
+from enum import Enum
+from pathlib import Path
+from ldap3 import Server, Connection, ALL, SAFE_SYNC, Tls
+from ldap3.core.exceptions import LDAPException
+from dataclasses import dataclass
 
 #PRODUCTION VALUES
 
-PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/"
-PRODUCTION_LDAP_SERVER = "ldaps://ldap.cilogon.org"
-PRODUCTION_LDAP_USER = "uid=readonly_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
-PRODUCTION_OSG_CO_ID = 7
-PRODUCTION_UNIX_CLUSTER_ID = 1
-PRODUCTION_LDAP_TARGET_ID = 6
+# PRODUCTION_ENDPOINT = "https://registry.cilogon.org/registry/"
+# PRODUCTION_OSG_CO_ID = 7
+# PRODUCTION_UNIX_CLUSTER_ID = 1
+# PRODUCTION_LDAP_TARGET_ID = 6
 
 #TEST VALUES
 
-TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/"
-TEST_LDAP_SERVER = "ldaps://ldap-test.cilogon.org"
-TEST_LDAP_USER ="uid=registry_user,ou=system,o=OSG,o=CO,dc=cilogon,dc=org"
-TEST_OSG_CO_ID = 8
-TEST_UNIX_CLUSTER_ID = 10
-TEST_LDAP_TARGET_ID = 9
+# TEST_ENDPOINT = "https://registry-test.cilogon.org/registry/"
+# TEST_OSG_CO_ID = 8
+# TEST_UNIX_CLUSTER_ID = 10
+# TEST_LDAP_TARGET_ID = 9
 
 # Value for the base of the exponential backoff
 TIMEOUT_BASE = 5
 MAX_ATTEMPTS = 5
 
 # HTTP return codes we shouldn't attempt to retry
 HTTP_NO_RETRY_CODES = {401, 403, 404, 405, 500}
+# LDAP Server Connection and Search Config, required keys
+class LDAP_CONFIG_KEYS(str, Enum):
+    LDAP_Server_URL = "LDAPServerURl"
+    LDAP_Search_Base = "SearchBase"
+    LDAP_User = "User"
+    LDAP_AuthTok_File = "AuthTokenFile"
+    
+    def __str__(self):
+        return f'{self.value}'
+
+LDAP_CONFIG_USAGE_MESSAGE = f"""
+LDAP CONNECTION CONFIG:
+File at LDAP_CONFIG_PATH should be in ini format, servers will be attempted in descending order.
+An example section of this config file follows:
+
+---
+
+[human_server_name]          # arbitrary human label for this server's config
+{LDAP_CONFIG_KEYS.LDAP_Server_URL} = ldaps://ldap-replica-1.osg.chtc.io      # URL to reach this LDAP server from
+{LDAP_CONFIG_KEYS.LDAP_Search_Base} = dc=osg-htc,dc=org                      # LDAP Search Base 
+{LDAP_CONFIG_KEYS.LDAP_User} = cn=readonly,ou=system,dc=osg-htc,dc=org       # full LDAP user DN 
+{LDAP_CONFIG_KEYS.LDAP_AuthTok_File} = /etc/ldap-secrets/osg-ldap/authtoken  # file containing authtoken for access
+
+---
+
+Config file should contain one such section per LDAP server to communicate with.
+"""
+
 
 GET    = "GET"
 PUT    = "PUT"
@@ -43,11 +72,18 @@ class Error(Exception):
     """Base exception class for all exceptions defined"""
     pass
 
-
 class URLRequestError(Error):
     """Class for exceptions due to not being able to fulfill a URLRequest"""
     pass
 
+class EmptyConfiguration(Error):
+    """Class for exceptions due to loading an empty Config file, or one where every section lacked the required keys"""
+    pass
+
+class NoLDAPResponse(Error):
+    """Class for exceptions due to being unable to get any request from any configured LDAP servers."""
+    pass
+
 
 class HTTPRequestError(URLRequestError):
     """Class for exceptions due to not being able to fulfill a HTTPRequest"""
@@ -81,6 +117,60 @@ def mkauthstr(user, passwd):
     return encodebytes(raw_authstr.encode()).decode().replace("\n", "")
 
 
+def get_ldap_authtok(ldap_authfile):
+    if ldap_authfile is not None:
+        ldap_authtok = open(ldap_authfile).readline().strip()
+    else:
+        raise PermissionError
+    return ldap_authtok
+
+
+def read_ldap_conffile(ldap_conffile_path):
+    config = configparser.ConfigParser(allow_no_value=True)
+
+    print(f"Attempting to read config from {ldap_conffile_path}")
+    config.read(ldap_conffile_path)
+    misconfigured_sections = list()
+    for section in config.sections():
+        for key in LDAP_CONFIG_KEYS:
+            # All servers must have all required keys for operation
+            if not config.has_option(section, key) or config.get(section, key) == "":
+                print(f"Section \"{section}\": required key \"{key}\" missing, ignoring section.")
+                misconfigured_sections.append(section)
+                break
+        # For-Else to only check key values if we know the required ones exist (i.e. we didn't break)
+        else:
+            # All server AuthTok files must exist, be files, and not be empty
+            token_path = Path(config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File))
+            try:
+                if not token_path.exists():
+                    print(f"Section \"{section}\": AuthToken File missing or non-file, ignoring section.")
+                    misconfigured_sections.append(section)
+                    continue
+                if token_path.stat().st_size == 0 :
+                    print(f"Section \"{section}\": AuthToken File is empty, ignoring section.")
+                    misconfigured_sections.append(section)
+                    continue
+            except OSError as e:
+                print(f"Section \"{section}\": Exception raised while checking AuthTok File: {e}, skipping section.")
+                misconfigured_sections.append(section)
+                continue
+
+    for section in misconfigured_sections:
+        print(f"Dropping section {section}")
+        config.remove_section(section)
+
+    # if their are no servers in the file that have all required keys
+    if len(config.sections()) == 0:
+        #when script needs to say LDAP Config required, raise a EmptyConfiguration error
+        #usage("LDAP Config File Required")
+        raise EmptyConfiguration(
+            f"Config file at {ldap_conffile_path} was empty or all sections lacked required keys."
+        )
+    print(f"Finished reading config from {ldap_conffile_path}")
+    return config
+
+
 def mkrequest(method, target, data, endpoint, authstr, **kw):
     url = os.path.join(endpoint, target)
     if kw:
@@ -223,16 +313,109 @@ def get_datalist(data, listname):
     return data[listname] if data else []
 
 
-def get_ldap_groups(ldap_server, ldap_user, ldap_authtok):
+class LDAPServer:
+    """ Wrapper class for LDAP searches. """
+    server: Server = None
+    connection: Connection = None
+
+    def __init__(self, ldap_server, ldap_user, ldap_authtok):
+        self.server = Server(ldap_server, get_info=ALL)
+        self.connection = Connection(self.server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
+
+    def search(self, ou, search_base, filter_str, attrs):
+        # simple paged search
+        # https://github.com/cannatag/ldap3/blob/7991e67d0a2fb2c1f9cbf832d110ad29fc378f9b/docs/manual/source/standard.rst#L4
+        # https://ldap3.readthedocs.io/en/latest/tutorial_searches.html#simple-paged-search
+        response = self.connection.extend.standard.paged_search(
+            f"ou={ou},{search_base}",
+            filter_str, 
+            attributes=attrs,
+            paged_size=500,
+            generator=True
+        )
+
+        return response
+
+
+# TODO:
+# do_ldap_fallback_search, get_ldap_groups, and get_ldap_active_users_and_groups should be a method of the LDAPSearch class
+# script calling this lib should init LDAPSearch, then call the method that asks for the info it wants.
+# Be able to feed in either one server's config to the LDAPSearch, or a conffile to parse with a list of >=1 LDAP servers to do fallback searches with.
+
+def do_ldap_fallback_search(search_ou, search_filter, attrs, ldap_config: configparser.ConfigParser):
+    response = None
+
+    if ldap_config == None:
+        raise EmptyConfiguration(
+            "Search Attempted with \"None\" config object."
+        )
+
+    for section in ldap_config.sections():
+        print(f"Attempting search with server {section}")
+        try:
+            server_url = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Server_URL)
+            search_base = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_Search_Base)
+            search_user = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_User)
+            authtok_file = ldap_config.get(section, LDAP_CONFIG_KEYS.LDAP_AuthTok_File)
+            authtok = get_ldap_authtok(authtok_file)
+
+            searcher = LDAPServer(ldap_server=server_url, ldap_user=search_user, ldap_authtok=authtok)
+            response = searcher.search(search_ou, search_base, search_filter, attrs)
+            
+            #If we get a response from one of the servers, we don't need to check the rest 
+            if not response is None:
+                print(f"Response found for server {section}.")
+                break
+        # Perm issue reading token file
+        except PermissionError as permError:
+            print(f"Permission Error when attempting search for {section}: {permError}.")
+        # Problem getting LDAP Response
+        except LDAPException as ldapError:
+            print(f"Exception occurred when attempting search for {section}: {ldapError}.")
+            continue
+
+    if response is None:
+        raise NoLDAPResponse(
+            f"No response found via LDAP servers: {[section for section in ldap_config]}."
+        )
+
+    return response
+
+
+def get_ldap_groups(config=None):
     ldap_group_osggids = set()
-    server = Server(ldap_server, get_info=ALL)
-    connection = Connection(server, ldap_user, ldap_authtok, client_strategy=SAFE_SYNC, auto_bind=True)
-    _, _, response, _ = connection.search("ou=groups,o=OSG,o=CO,dc=cilogon,dc=org", "(cn=*)", attributes=ALL_ATTRIBUTES)
+
+    response = do_ldap_fallback_search(
+        search_ou="groups",
+        search_filter="(cn=*)",
+        attrs=["gidNumber"],
+        ldap_config=config
+    )
+
     for group in response:
         ldap_group_osggids.add(group["attributes"]["gidNumber"])
     return ldap_group_osggids
 
 
+def get_ldap_active_users_and_groups(filter_group_name=None, config=None):
+    """ Retrieve a dictionary of active users from LDAP, with their group memberships. """
+    ldap_active_users = dict()
+    filter_str = ("(isMemberOf=CO:members:active)" if filter_group_name is None 
+                  else f"(&(isMemberOf={filter_group_name})(isMemberOf=CO:members:active))")
+
+    response = do_ldap_fallback_search(
+        search_ou="people",
+        search_filter=filter_str,
+        attrs=["employeeNumber", "isMemberOf"],
+        ldap_config=config
+    )
+
+    for person in response:
+        ldap_active_users[person["attributes"]["employeeNumber"]] = person["attributes"].get("isMemberOf", [])
+
+    return ldap_active_users
+
+
 def identifier_from_list(id_list, id_type):
     id_type_list = [id["Type"] for id in id_list]
     try:
@@ -324,6 +507,7 @@ def provision_group(gid, provision_target, endpoint, authstr):
     }
     return call_api3(POST, path, data, endpoint, authstr)
 
+
 def provision_group_members(gid, prov_id, endpoint, authstr):
     data = {
         "RequestType" : "CoPersonProvisioning",

create_project.py

@@ -152,4 +152,3 @@ def main(args):
     except (RuntimeError, urllib.error.HTTPError) as e:
         print(e, file=sys.stderr)
         sys.exit(1)
-