Merge pull request #33 from Metasemi/morr-cxmap

Pull Mike's extensions work into v1

Author: phil-levy

docs/User Sync Guide.docx

@@ -205,15 +205,10 @@ def get_directory_groups(self):
 
             dashboard_groups_config = item.get_list_config('dashboard_groups')
             for dashboard_group in dashboard_groups_config.iter_values(types.StringTypes):
-                parts = dashboard_group.split(GROUP_NAME_DELIMITER)
-                group_name = parts.pop()
-                organization_name = GROUP_NAME_DELIMITER.join(parts)
-                if (len(organization_name) == 0):
-                    organization_name = user_sync.rules.OWNING_ORGANIZATION_NAME
-                if (len(group_name) == 0):
+                group = self.create_dashboard_group(dashboard_group)
+                if (group is None):
                     validation_message = 'Bad dashboard group: "%s" in directory group: "%s"' % (dashboard_group, directory_group)
-                    raise user_sync.error.AssertionException(validation_message)                    
-                group = user_sync.rules.Group(group_name, organization_name)
+                    raise user_sync.error.AssertionException(validation_message)
                 groups.append(group)
 
         return adobe_groups_by_directory_group
@@ -324,6 +319,40 @@ def get_rule_options(self):
         max_deletions_per_run = limits_config.get_int('max_deletions_per_run')
         max_missing_users = limits_config.get_int('max_missing_users')
 
+        after_mapping_hook = None
+        extended_attributes = None
+        extensions_config = self.main_config.get_list_config('extensions', True)
+        if (extensions_config != None):
+            for extension_config in extensions_config.iter_dict_configs():
+                context = extension_config.get_string('context')
+                if context == 'per-user':
+                    if (after_mapping_hook == None):
+                        after_mapping_hook_text = extension_config.get_string('after_mapping_hook')
+                        if (after_mapping_hook_text is not None):
+                            after_mapping_hook = compile(after_mapping_hook_text, '<per-user after-mapping-hook>', 'exec')
+                            extended_attributes = extension_config.get_list('extended_attributes')
+
+                            # [TODO morr 2017-02-27]: Do we really need to pre-create extended dashboard groups here? Or
+                            # could it be done on the fly, when they're encountered in values returned from hook code? If
+                            # the latter, we could do the customer a big favor by not requiring them to be declared in the
+                            # extension config.
+                            #
+                            # This should be revisited once the Complex Mapping feature as a whole is working and has been
+                            # thoroughly tested.
+                            #
+                            for extended_dashboard_group in extension_config.get_list('extended_dashboard_groups'):
+                                group = self.create_dashboard_group(extended_dashboard_group)
+                                if (group is None):
+                                    validation_message = 'Bad dashboard group: "%s" in extension with context "%s"' % (extended_dashboard_group, context)
+                                    raise user_sync.error.AssertionException(validation_message)
+
+                        else:
+                            self.logger.warning("No valid hook found in extension with context '%s'; extension ignored")
+                    else:
+                        self.logger.warning("Duplicate extension context '%s' ignored", context)
+                else:
+                    self.logger.warning("Unrecognized extension context '%s' ignored", context)
+        
         options = self.options
         result = {
             'directory_group_filter': options['directory_group_filter'],
@@ -336,7 +365,9 @@ def get_rule_options(self):
             'remove_nonexistent_users': options['remove_nonexistent_users'],
             'default_country_code': default_country_code,
             'max_deletions_per_run': max_deletions_per_run,
-            'max_missing_users': max_missing_users
+            'max_missing_users': max_missing_users,
+            'after_mapping_hook': after_mapping_hook,
+            'extended_attributes': extended_attributes,
         }
         return result
 
@@ -351,7 +382,23 @@ def create_dashboard_options(self, connector_config_sources, owner):
                 connector_config['enterprise'] = new_enterprise_section
 
         return connector_config
-    
+
+    def create_dashboard_group(self, dashboard_group_qualified_name):
+        parts = dashboard_group_qualified_name.split(GROUP_NAME_DELIMITER)
+        group_name = parts.pop()
+        organization_name = GROUP_NAME_DELIMITER.join(parts)
+        if (len(organization_name) == 0):
+            organization_name = user_sync.rules.OWNING_ORGANIZATION_NAME
+
+        group = None
+        if (len(group_name) > 0):
+            # check for existing group in case someone mistakenly declared an extended group that's already a mapping target
+            group = user_sync.rules.Group.get_dashboard_group(group_name, organization_name)
+            if group is None:
+                group = user_sync.rules.Group(group_name, organization_name)
+
+        return group
+
     def check_unused_config_keys(self):
         directory_connectors_config = self.get_directory_connector_configs()
         self.main_config.report_unused_values(self.logger, [directory_connectors_config])

docs/User Sync Guide.pdf

@@ -40,10 +40,11 @@ def initialize(self, options = {}):
         '''
         self.state = self.implementation.connector_initialize(options)
 
-    def load_users_and_groups(self, groups):
+    def load_users_and_groups(self, groups, extended_attributes=None):
         '''
         :type groups: list(str)
+        :type extended_attributes: list(str)
         :rtype (bool, iterable(dict))
         '''
-        return self.implementation.connector_load_users_and_groups(self.state, groups)        
+        return self.implementation.connector_load_users_and_groups(self.state, groups, extended_attributes)
 

user_sync/config.py

@@ -37,12 +37,16 @@ def connector_initialize(options):
     state = CSVDirectoryConnector(options)
     return state
 
-def connector_load_users_and_groups(state, groups):
+def connector_load_users_and_groups(state, groups, extended_attributes):
     '''
     :type state: CSVDirectoryConnector
     :type groups: list(str)
+    :type extended_attributes: list(str)
     :rtype (bool, iterable(dict))
     '''
+
+    # CSV supports arbitrary aka "extended" attrs by default, so the value of extended_attributes has no impact on this particular connector
+
     return state.load_users_and_groups(groups)
 
 class CSVDirectoryConnector(object):

user_sync/connector/directory.py

@@ -39,13 +39,14 @@ def connector_initialize(options):
     connector = LDAPDirectoryConnector(options)
     return connector
 
-def connector_load_users_and_groups(state, groups):
+def connector_load_users_and_groups(state, groups, extended_attributes):
     '''
     :type state: LDAPDirectoryConnector
     :type groups: list(str)
+    :type extended_attributes: list(str)
     :rtype (bool, iterable(dict))
     '''
-    return state.load_users_and_groups(groups)
+    return state.load_users_and_groups(groups, extended_attributes)
 
 class LDAPDirectoryConnector(object):
     name = 'ldap'
@@ -103,9 +104,10 @@ def __init__(self, caller_options):
         self.connection = connection
         logger.info('Connected')            
 
-    def load_users_and_groups(self, groups):
+    def load_users_and_groups(self, groups, extended_attributes):
         '''
         :type groups: list(str)
+        :type extended_attributes: list(str)
         :rtype (bool, iterable(dict))
         '''
         options = self.options
@@ -125,12 +127,12 @@ def load_users_and_groups(self, groups):
 
         self.user_by_dn = user_by_dn = {}
         self.user_by_uid = user_by_uid = {}
-        for user_dn, user in self.iter_users(users_filter):
+        for user_dn, user in self.iter_users(users_filter, extended_attributes):
             uid = user.get('uid')
             if (uid != None):
                 user_by_uid[uid] = user
             user_by_dn[user_dn] = user
-        
+
         self.logger.info('Total users loaded: %d', len(user_by_dn))
 
         for group in groups:
@@ -250,15 +252,18 @@ def iter_ldap_group_members(self, group):
                 for attribute_value in attribute_values:
                     yield (attribute, attribute_value)
 
-    def iter_users(self, users_filter):
+    def iter_users(self, users_filter, extended_attributes):
         options = self.options
         base_dn = options['base_dn']
-        
-        user_attribute_names = ["givenName", "sn", "c", "uid"]    
+
+        user_attribute_names = ["givenName", "sn", "c", "uid"]
         user_attribute_names.extend(self.user_email_formatter.get_attribute_names())
         user_attribute_names.extend(self.user_username_formatter.get_attribute_names())
         user_attribute_names.extend(self.user_domain_formatter.get_attribute_names())
 
+        extended_attributes = list(set(extended_attributes) - set(user_attribute_names))
+        user_attribute_names.extend(extended_attributes)
+
         result_iter = self.iter_search_result(base_dn, ldap.SCOPE_SUBTREE, users_filter, user_attribute_names)
         for dn, record in result_iter:
             if (dn == None):
@@ -269,35 +274,53 @@ def iter_users(self, users_filter):
                 if (last_attribute_name != None):
                     self.logger.warn('No email attribute: %s for dn: %s', last_attribute_name, dn)
                 continue
+
+            source_attributes = {}
 
             user = user_sync.connector.helper.create_blank_user()
+            source_attributes['email'] = email
             user['email'] = email
-                
+
             username, last_attribute_name = self.user_username_formatter.generate_value(record)
+            source_attributes['username'] = username
             if (username == None and last_attribute_name != None):
                 self.logger.info('No username attribute: %s for dn: %s', last_attribute_name, dn)    
             user['username'] = username if username != None else email
-                    
+
             domain, last_attribute_name = self.user_domain_formatter.generate_value(record)
+            source_attributes['domain'] = domain
             if (domain != None):
                 user['domain'] = domain
             elif (last_attribute_name != None):
                 self.logger.info('No domain attribute: %s for dn: %s', last_attribute_name, dn)    
 
             given_name_value = LDAPValueFormatter.get_attribute_value(record, 'givenName')
-            if (given_name_value != None):   
+            source_attributes['givenName'] = given_name_value
+            if (given_name_value != None):
                 user['firstname'] = given_name_value
             sn_value = LDAPValueFormatter.get_attribute_value(record, 'sn')
+            source_attributes['sn'] = sn_value
             if sn_value != None:
                 user['lastname'] = sn_value
             c_value = LDAPValueFormatter.get_attribute_value(record, 'c')
+            source_attributes['c'] = c_value
             if c_value != None:
                 user['country'] = c_value
-                
+
             uid = LDAPValueFormatter.get_attribute_value(record, 'uid')
+            source_attributes['uid'] = uid
             if (uid != None):
                 user['uid'] = uid
-            
+
+            if extended_attributes is not None:
+                for extended_attribute in extended_attributes:
+                    extended_attribute_value = LDAPValueFormatter.get_attribute_value(record, extended_attribute)
+                    source_attributes[extended_attribute] = extended_attribute_value
+
+            # [TODO morr 2017-02-26]: Could be omitted if no hook; worth considering?
+            # [TODO morr 2017-02-28]: Is the copy necessary? Could just assign I think
+            user['source_attributes'] = source_attributes.copy()
+
             yield (dn, user)
 
     def iter_search_result(self, base_dn, scope, filter_string, attributes):