Survive bad json, more statistics (#155)

Author: Barthelemy

Framework/script/RepoCleaner/1_per_hour.py

@@ -4,6 +4,7 @@
 
 from Ccdb import Ccdb, ObjectVersion
 
+
 def process(ccdb: Ccdb, object_path: str, delay: int):
     '''
     Process this deletion rule on the object. We use the CCDB passed by argument.
@@ -15,6 +16,7 @@ def process(ccdb: Ccdb, object_path: str, delay: int):
     :param ccdb: the ccdb in which objects are cleaned up.
     :param object_path: path to the object, or pattern, to which a rule will apply.
     :param delay: the grace period during which a new object is never deleted.
+    :return a dictionary with the number of deleted, preserved and updated versions. Total = deleted+preserved.
     '''
 
     logging.debug(f"Plugin 1_per_hour processing {object_path}")
@@ -24,11 +26,13 @@ def process(ccdb: Ccdb, object_path: str, delay: int):
     last_preserved: ObjectVersion = None
     preservation_list: List[ObjectVersion] = []
     deletion_list: List[ObjectVersion] = []
+    update_list: List[ObjectVersion] = []
     for v in versions:
         if last_preserved == None or last_preserved.validFromAsDatetime < v.validFromAsDatetime - timedelta(hours=1):
             # first extend validity of the previous preserved (should we take into account the run ?)
             if last_preserved != None:
                 ccdb.updateValidity(last_preserved, last_preserved.validFrom, str(int(v.validFrom) - 1))
+                update_list.append(last_preserved)
             last_preserved = v
             preservation_list.append(v)
         else:
@@ -45,9 +49,17 @@ def process(ccdb: Ccdb, object_path: str, delay: int):
     for v in preservation_list:
         logging.debug(f"   {v}")
 
+    logging.debug("updated : ")
+    for v in update_list:
+        logging.debug(f"   {v}")
+
+    return {"deleted" : len(deletion_list), "preserved": len(preservation_list), "updated" : len(update_list)}
+
+
 def main():
     ccdb = Ccdb('http://ccdb-test.cern.ch:8080')
     process(ccdb, "asdfasdf/example", 60)
 
-if __name__== "__main__": # to be able to run the test code above when not imported.
+
+if __name__ == "__main__":  # to be able to run the test code above when not imported.
     main()

Framework/script/RepoCleaner/Ccdb.py

@@ -74,9 +74,13 @@ def getVersionsList(self, object_path: str) -> List[ObjectVersion]:
         headers = {'Accept':'application/json'}
         r = requests.get(url_browse_all_versions, headers=headers)
         r.raise_for_status()
-        json = r.json()
+        try:
+            json_result = r.json(strict=False) # to survive bad characters in the strings of the json
+        except ValueError as e:
+            print(f"Error while reading json for object {object_path} from CCDB: {e}")
+            exit(1)
         versions = []
-        for object_path in json['objects']:
+        for object_path in json_result['objects']:
             version = ObjectVersion(path=object_path['path'], uuid=object_path['id'], validFrom=object_path['validFrom'], validTo=object_path['validUntil'])
             versions.insert(0, version)
         return versions
@@ -87,9 +91,8 @@ def deleteVersion(self, version: ObjectVersion):
         Delete the specified version of an object. 
         :param version: The version of the object to delete, as an instance of ObjectVersion.
         '''
-        logging.debug(f"Delete version: {version}")
         url_delete = self.url + '/' + version.path + '/' + version.validFrom + '/' + version.uuid
-        logging.info(f"Delete version at url {url_delete}")
+        logging.debug(f"Delete version at url {url_delete}")
         try:
             r = requests.delete(url_delete)
             r.raise_for_status()
@@ -110,7 +113,7 @@ def updateValidity(self, version: ObjectVersion, validFrom: str, validTo: str):
             logging.debug("The new timestamp for validTo is identical to the existing one. Skipping.")
             return
         url_update_validity = self.url + '/' + version.path + '/' + validFrom + '/' + validTo
-        logging.info(f"Update end limit validity of {version.path} from {version.validTo} to {validTo}")
+        logging.debug(f"Update end limit validity of {version.path} from {version.validTo} to {validTo}")
         try:
             r = requests.put(url_update_validity)
             r.raise_for_status()

Framework/script/RepoCleaner/config.yaml

@@ -1,5 +1,5 @@
 Rules:
-  - object_path: asdfasdf/.* # Path in the CCDB to a certain object
+  - object_path: .* # Path in the CCDB to a certain object
     delay: 1440              # Delay in minutes during which a new object is not touched. (1 day)
     policy: 1_per_hour       # name of the policy to apply, must correspond to a python script.
 #  - object_path: QcTask/example

Framework/script/RepoCleaner/repoCleaner.py

@@ -127,16 +127,18 @@ def main():
     # For each object call the first matching rule
     logging.info("Loop through the objects and apply first matching rule.")
     for object_path in paths:
+        logging.info(f"Processing {object_path}")
         # Take the first matching rule, if any
         rule = findMatchingRule(rules, object_path);
         if rule == None:
             continue
 
         # Apply rule on object (find the plug-in script and apply)
         module = __import__(rule.policy)
-        module.process(ccdb, object_path, int(rule.delay))
+        stats = module.process(ccdb, object_path, int(rule.delay))
+        logging.info(f"{rule.policy} applied on {object_path}: {stats}")
 
-    logging.info(f"done (deleted: {ccdb.counter_deleted}, updated: {ccdb.counter_validity_updated})")
+    logging.info(f" *** DONE *** (total deleted: {ccdb.counter_deleted}, total updated: {ccdb.counter_validity_updated})")
 
 if __name__ == "__main__":  # to be able to run the test code above when not imported.
     main()