Merge branch 'v1' of https://github.com/adobe-apiplatform/user-sync.py into v1

Author: phil-levy

examples/example.user-sync-config.yml

@@ -66,6 +66,10 @@ directory:
   # Default is:
   # user_identity_type: enterpriseID
 
+limits:
+    max_deletions_per_run: 10    # if --remove-nonexistent-users is specified, this is the most users that will be removed.  Others will be left for a later run.  A critical message will be logged.
+    max_missing_users: 200       # if more than this number of user accounts are not found in the directory, user sync will abort with an error and a critical message will be logged.
+
 logging:
   # specifies whether you wish to generate a log file
   # 'True' or 'False'

tests/config_test.py

@@ -95,6 +95,8 @@ def test_get_rule_options(self, mock_id_type,mock_get_dict,mock_get_string):
         self.assertEquals(self.conf_load.get_rule_options(), {'username_filter_regex': None,
                                                               'update_user_info': True,
                                                               'manage_groups': True,
+                                                              'max_deletions_per_run': 1,
+                                                              'max_missing_users': 1,
                                                               'new_account_type': 'new_acc',
                                                               'directory_group_filter': None,
                                                               'default_country_code': 'test',

tests/helper.py

@@ -85,4 +85,7 @@ def create_action_manager():
 
 class MockGetString():
     def get_string(self,test1,test2):
-        return 'test'
+        return 'test'
+
+    def get_int(self,test1):
+        return 1

user_sync/app.py

@@ -275,7 +275,7 @@ def main():
 
     except user_sync.error.AssertionException as e:
         if (not e.is_reported()):
-            logger.error(e.message)
+            logger.critical(e.message)
             e.set_reported()    
     except:
         try:

user_sync/config.py

@@ -319,7 +319,11 @@ def get_rule_options(self):
         if (new_account_type == None):
             new_account_type = user_sync.identity_type.ENTERPRISE_IDENTITY_TYPE
             self.logger.warning("Assuming the identity type for users is: %s", new_account_type)
-        
+
+        limits_config = self.main_config.get_dict_config('limits')
+        max_deletions_per_run = limits_config.get_int('max_deletions_per_run')
+        max_missing_users = limits_config.get_int('max_missing_users')
+
         options = self.options
         result = {
             'directory_group_filter': options['directory_group_filter'],
@@ -330,7 +334,9 @@ def get_rule_options(self):
             'remove_user_key_list': options['remove_user_key_list'],
             'remove_list_output_path': options['remove_list_output_path'],
             'remove_nonexistent_users': options['remove_nonexistent_users'],
-            'default_country_code': default_country_code
+            'default_country_code': default_country_code,
+            'max_deletions_per_run': max_deletions_per_run,
+            'max_missing_users': max_missing_users
         }
         return result
 
@@ -501,7 +507,10 @@ def get_dict(self, key, none_allowed = False):
 
     def get_string(self, key, none_allowed = False):
         return self.get_value(key, types.StringTypes, none_allowed)
-    
+
+    def get_int(self, key, none_allowed = False):
+        return self.get_value(key, types.IntType, none_allowed)
+
     def get_bool(self, key, none_allowed = False):
         return self.get_value(key, types.BooleanType, none_allowed)
 

user_sync/rules.py

@@ -44,7 +44,9 @@ def __init__(self, caller_options):
             'remove_user_key_list': None,
             'remove_list_output_path': None,
             'remove_nonexistent_users': False,
-            'default_country_code': None
+            'default_country_code': None,
+            'max_deletions_per_run': None,
+            'max_missing_users': None
         }
         options.update(caller_options)        
         self.options = options        
@@ -234,14 +236,29 @@ def process_orphaned_dashboard_users(self):
         remove_list_output_path = options['remove_list_output_path']
         remove_nonexistent_users = options['remove_nonexistent_users']
 
+        max_deletions_per_run = options['max_deletions_per_run']
+        max_missing_users = options['max_missing_users']
+
         orphaned_federated_dashboard_users = list(self.iter_orphaned_federated_dashboard_users())
         self.logger.info('Federated orphaned users to be removed: %s', [self.get_dashboard_user_key(dashboard_user) for dashboard_user in orphaned_federated_dashboard_users])        
 
+        number_of_orphaned_dashboard_users = len(orphaned_federated_dashboard_users)
+
         if (remove_list_output_path != None):
             self.logger.info('Writing remove list to: %s', remove_list_output_path)
             self.write_remove_list(remove_list_output_path, orphaned_federated_dashboard_users)
         elif (remove_nonexistent_users):
+            if number_of_orphaned_dashboard_users > max_missing_users:
+                raise user_sync.error.AssertionException(
+                    'Unable to process orphaned users, as number of users (%s) is larger than max_missing_users setting' % number_of_orphaned_dashboard_users)
+            orphan_count = 0
             for dashboard_user in orphaned_federated_dashboard_users:
+                orphan_count += 1
+                if orphan_count > max_deletions_per_run:
+                    self.logger.critical('Only processing %d of the %d orphaned users ' +
+                                         'due to max_deletions_per_run setting', max_deletions_per_run,
+                                         number_of_orphaned_dashboard_users)
+                    break
                 user_key = self.get_dashboard_user_key(dashboard_user)
                 remove_user_key_list.add(user_key)