Change date_imported to date_collected for comparison

Signed-off-by: Tushar Goel <tushar.goel.dav@gmail.com>

Author: TG1999

vulnerabilities/pipelines/recompute_content_ids.py

@@ -99,7 +99,7 @@ def process_advisories(
         log(f"Running function in single process", level=logging.INFO)
         for advisory_ids in progress.iter(advisory_batches):
             progress.log_progress()
-            advisory_func(advisory_ids=advisory_ids, logger=log)
+            advisory_func(advisory_ids=advisory_ids, log=log)
         return
 
     log(
@@ -147,23 +147,23 @@ def get_advisory_batches(advisories, batch_size=1000, log=None):
         yield advisory_ids
 
 
-def recompute_content_ids(advisory_ids, logger):
+def recompute_content_ids(advisory_ids, log):
     """
     Recompute content IDs for all `advisory_ids`.
     """
     advisories = Advisory.objects.exclude(unique_content_id__length=64).filter(id__in=advisory_ids)
     total_count = advisories.count()
 
     if not total_count:
-        logger("No advisories need content ID recomputation", level=logging.INFO)
+        log("No advisories need content ID recomputation", level=logging.INFO)
         return
 
-    logger(f"Recomputing content IDs for {total_count} advisories", level=logging.INFO)
+    log(f"Recomputing content IDs for {total_count} advisories", level=logging.INFO)
 
     progress = LoopProgress(
         total_iterations=total_count,
         progress_step=total_count // 100,
-        logger=logger,
+        logger=log,
     )
 
     with transaction.atomic():
@@ -181,8 +181,8 @@ def recompute_content_ids(advisory_ids, logger):
                 ["unique_content_id"],
                 batch_size=len(advisories_to_update),
             )
-            if logger:
-                logger(
+            if log:
+                log(
                     f"Updated content IDs for {len(advisories_to_update)} advisories",
                     level=logging.INFO,
                 )

vulnerabilities/pipelines/remove_duplicate_advisories.py

@@ -8,6 +8,7 @@
 #
 
 import logging
+import traceback
 from itertools import groupby
 
 from aboutcode.pipeline import LoopProgress
@@ -19,7 +20,7 @@
 from vulnerabilities.pipelines.recompute_content_ids import process_advisories
 
 
-def remove_duplicates_batch(advisory_ids, logger=None):
+def remove_duplicates_batch(advisory_ids, log=None):
     """
     Process a batch of advisories to remove duplicates.
     Keep only the oldest advisory for each content ID.
@@ -37,31 +38,33 @@ def remove_duplicates_batch(advisory_ids, logger=None):
                 key=lambda x: x.unique_content_id,
             )
 
-            progress = LoopProgress(total_iterations=advisories.count(), logger=logger)
+            progress = LoopProgress(total_iterations=advisories.count(), logger=log)
 
             for content_id, group_advisories in progress.iter(advisories_by_content_id):
                 group_advisories = list(group_advisories)
 
                 if len(group_advisories) <= 1:
                     continue
 
-                logger(
+                log(
                     f"Found {len(group_advisories)} duplicates for content ID {content_id}",
+                    level=logging.INFO,
                 )
 
-                oldest = min(group_advisories, key=lambda x: x.date_imported)
+                oldest = min(group_advisories, key=lambda x: x.date_collected)
 
                 advisory_ids_to_delete = [adv.id for adv in group_advisories if adv.id != oldest.id]
                 if advisory_ids_to_delete:
                     Advisory.objects.filter(id__in=advisory_ids_to_delete).delete()
-                    logger(
+                    log(
                         f"Kept advisory {oldest.id} and removed "
                         f"{len(advisory_ids_to_delete)} duplicates for content ID {content_id}",
+                        level=logging.INFO,
                     )
 
     except Exception as e:
-        logger(
-            f"Error processing batch of advisories: {e}",
+        log(
+            f"Error removing duplicates for batch of advisories: {traceback.format_exc()}",
             level=logging.ERROR,
         )