Script to delete objects not in a list of runs (#1680)

* script to delete objects not in a list of runs

* comment

* Update test_MultiplePerRun.py

Author: Barthelemy

Framework/script/RepoCleaner/ReleaseNotes.md

@@ -3,6 +3,7 @@
 1.3
 - Add new tool `o2-qc-repo-find-objects-not-updated` to find all the objects under a path that did not get a new
   version in the past X days. 
+- Add new tool to delete objects not belonging to a list of runs.
 
 1.2
 

Framework/script/RepoCleaner/qcrepocleaner/binUtils.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+
+import logging
+import sys
+
+
+def prepare_main_logger():
+    logger = logging.getLogger()
+    # Logging (split between stderr and stdout)
+    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
+    h1 = logging.StreamHandler(sys.stdout)
+    h1.setLevel(logging.DEBUG)
+    h1.addFilter(lambda record: record.levelno <= logging.INFO) # filter out everything that is above INFO level
+    h1.setFormatter(formatter)
+    logger.addHandler(h1)
+    h2 = logging.StreamHandler(sys.stderr)
+    h2.setLevel(logging.WARNING)     # take only warnings and error logs
+    h2.setFormatter(formatter)
+    logger.addHandler(h2)

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-not-in-runs

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+
+import logging
+import argparse
+import dryable
+from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
+from qcrepocleaner.binUtils import prepare_main_logger
+import csv
+
+
+def parseArgs():
+    """Parse the arguments passed to the script."""
+    logging.info("Parsing arguments")
+    parser = argparse.ArgumentParser(description='Remove all objects in a given path, if they dont belong to any of '
+                                                 'the runs. The runs are provided in a csv file whose format is the '
+                                                 'one from the bookkeeping export.')
+    parser.add_argument('--url', dest='url', action='store', help='URL of the CCDB, with http[s]://', required=True)
+    parser.add_argument('--log-level', dest='log_level', action='store', default="20",
+                        help='Log level (CRITICAL->50, ERROR->40, WARNING->30, INFO->20,DEBUG->10)')
+    parser.add_argument('--dry-run', action='store_true',
+                        help='Dry run, no actual deletion nor modification to the CCDB.')
+    parser.add_argument('--path', dest='path', action='store', default="",
+                        help='Clean this path (without initial slash and without .* at the end, e.g. qc/TST/MO/Bob)',
+                        required=True)
+    parser.add_argument('--runs-csv-file', dest='runs_file', action='store', help='A csv file with a header and the '
+                                                                                  'column `runNumber` contains the run',
+                        required=True)
+    parser.add_argument('--one-by-one', action='store_true', help='Ask confirmation for each deletion')
+    parser.add_argument('--print-list', action='store_true', help='Only print the list of objects that would be deleted')
+    args = parser.parse_args()
+    dryable.set(args.dry_run)
+    logging.info(args)
+    return args
+
+
+def run(args):
+    ccdb = Ccdb(args.url)
+
+    file = open(args.runs_file)
+    csvreader = csv.DictReader(file)
+    list_runs = []
+    for row in csvreader:
+        list_runs.append(row["runNumber"])
+    logging.debug(f"List of runs in CSV: {list_runs}")
+
+    versions = ccdb.getVersionsList(args.path + "/.*")
+    nb_deleted = 0
+    for v in versions:
+        logging.debug(f"Processing {v}")
+        run_number = v.metadata["RunNumber"]
+        if run_number is not None and list_runs.count(run_number) == 0:
+            logging.info(f"Ready to delete {v}")
+            if args.one_by_one:
+                answer = input("  Continue? y/n\n  ")
+                if answer.lower() in ["y", "yes"]:
+                    ccdb.deleteVersion(v)
+                    nb_deleted += 1
+                elif answer.lower() in ["n", "no"]:
+                    logging.info("   skipping")
+                else:
+                    logging.error("   wrong input, skipping")
+            else:
+                if not args.print_list:
+                    ccdb.deleteVersion(v)
+                nb_deleted += 1
+
+    logging.info(f"Deleted items: {nb_deleted}")
+
+
+# ****************
+# We start here !
+# ****************
+
+def main():
+
+    prepare_main_logger()
+
+    # Parse arguments
+    args = parseArgs()
+    logging.getLogger().setLevel(int(args.log_level))
+
+    run(args)
+
+
+if __name__ == "__main__":
+    main()

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-objects-in-runs

@@ -5,7 +5,7 @@ import argparse
 import dryable
 import time
 from qcrepocleaner.Ccdb import Ccdb, ObjectVersion
-import sys
+from qcrepocleaner.binUtils import prepare_main_logger
 import csv
 
 
@@ -34,21 +34,6 @@ def parseArgs():
     return args
 
 
-def prepare_main_logger():
-    logger = logging.getLogger()
-    # Logging (split between stderr and stdout)
-    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
-    h1 = logging.StreamHandler(sys.stdout)
-    h1.setLevel(logging.DEBUG)
-    h1.addFilter(lambda record: record.levelno <= logging.INFO) # filter out everything that is above INFO level
-    h1.setFormatter(formatter)
-    logger.addHandler(h1)
-    h2 = logging.StreamHandler(sys.stderr)
-    h2.setLevel(logging.WARNING)     # take only warnings and error logs
-    h2.setFormatter(formatter)
-    logger.addHandler(h2)
-
-
 def run(args):
     ccdb = Ccdb(args.url)
 

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-delete-time-interval

@@ -3,9 +3,8 @@
 import logging
 import argparse
 import dryable
-import time
 from qcrepocleaner.Ccdb import Ccdb
-import sys
+from qcrepocleaner.binUtils import prepare_main_logger
 
 def parseArgs():
     """Parse the arguments passed to the script."""
@@ -30,21 +29,6 @@ def parseArgs():
     return args
 
 
-def prepare_main_logger():
-    logger = logging.getLogger()
-    # Logging (split between stderr and stdout)
-    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
-    h1 = logging.StreamHandler(sys.stdout)
-    h1.setLevel(logging.DEBUG)
-    h1.addFilter(lambda record: record.levelno <= logging.INFO) # filter out everything that is above INFO level
-    h1.setFormatter(formatter)
-    logger.addHandler(h1)
-    h2 = logging.StreamHandler(sys.stderr)
-    h2.setLevel(logging.WARNING)     # take only warnings and error logs
-    h2.setFormatter(formatter)
-    logger.addHandler(h2)
-
-
 def run(args):
     ccdb = Ccdb(args.url)
     path = args.path+"/" if args.only_path_no_subdir else args.path + "/.*"
@@ -80,12 +64,12 @@ def run(args):
 
     logging.info(f"Deleted items: {len(versions)}")
 
+
 # ****************
 # We start here !
 # ****************
 
 def main():
-    start_time = time.time()
     prepare_main_logger()
 
     # Parse arguments

Framework/script/RepoCleaner/qcrepocleaner/o2-qc-repo-move-objects

@@ -4,7 +4,7 @@ import logging
 import argparse
 import dryable
 from qcrepocleaner.Ccdb import Ccdb
-import sys
+from qcrepocleaner.binUtils import prepare_main_logger
 import re
 
 
@@ -29,21 +29,6 @@ def parse_args():
     return args
 
 
-def prepare_main_logger():
-    logger = logging.getLogger()
-    # Logging (split between stderr and stdout)
-    formatter = logging.Formatter(fmt='%(asctime)s - %(levelname)s - %(message)s', datefmt='%d-%b-%y %H:%M:%S')
-    h1 = logging.StreamHandler(sys.stdout)
-    h1.setLevel(logging.DEBUG)
-    h1.addFilter(lambda record: record.levelno <= logging.INFO) # filter out everything that is above INFO level
-    h1.setFormatter(formatter)
-    logger.addHandler(h1)
-    h2 = logging.StreamHandler(sys.stderr)
-    h2.setLevel(logging.WARNING)     # take only warnings and error logs
-    h2.setFormatter(formatter)
-    logger.addHandler(h2)
-
-
 def run(args):
     ccdb = Ccdb(args.url)
 

Framework/script/RepoCleaner/tests/test_MultiplePerRun.py

@@ -77,19 +77,18 @@ def test_5_runs(self):
 
         # Prepare data
         test_path = self.path + "/test_5_runs"
-        self.prepare_data(test_path, [60, 120, 190, 240, 300], [60, 120, 190, 240, 24*60], 123)
+        self.prepare_data(test_path, [3, 3, 3, 3, 3], [60, 120, 190, 240, 24*60], 123)
 
         stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1,
                                        to_timestamp=self.in_ten_years, extra_params=self.extra)
-
         self.assertEqual(stats["deleted"], 60+120+190+240+300-18)
         self.assertEqual(stats["preserved"], 18)
         self.assertEqual(stats["updated"], 0)
-
+        
         # and now re-run it to make sure we preserve the state
         stats = multiple_per_run.process(self.ccdb, test_path, delay=60*24, from_timestamp=1,
-                                         to_timestamp=self.in_ten_years, extra_params=self.extra)
-
+                                          to_timestamp=self.in_ten_years, extra_params=self.extra)
+        
         self.assertEqual(stats["deleted"], 0)
         self.assertEqual(stats["preserved"], 18)
         self.assertEqual(stats["updated"], 0)