Merge pull request aboutcode-org#2309 from aboutcode-org/add_datasource_name

Add datasource ID

Author: TG1999

CHANGELOG.rst

@@ -4,7 +4,8 @@ Release notes
 next release
 ---------------------
 
-- WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories. 
+- WARNING: Vulnerablecode V1 API and UI has stopped supporting Ubuntu OVAL advisories, please shift to V3 API for new Ubuntu advisories.
+- Add attribute ``pipeline_id`` to AdvisoryV2 to track the pipeline that created the advisory, also rename existing ``datasource_id`` and AVIDs. 
 
 Version v38.6.0
 ---------------------

PIPELINES-AVID.rst

@@ -1,11 +1,13 @@
-.. list-table:: Pipeline AVID Mapping
+.. list-table:: Pipeline Advisory UID Mapping
    :header-rows: 1
    :widths: 35 65
 
    * - pipeline name
-     - AVID
+     - Advisory UID
+     - datasource name
    * - alpine_linux_importer_v2
      - {package_name}/{distroversion}/{version}/{vulnerability_id}
+     - alpine_linux
    * - aosp_dataset_fix_commits
      - CVE ID of the record
    * - apache_httpd_importer_v2

docs/source/api_v3_usage.rst

@@ -143,6 +143,7 @@ Example response:
                 "affected_by_vulnerabilities": [
                     {
                     "advisory_id": "GHSA-g5vw-3h65-2q3v",
+                    "advisory_uid": "ghsa/g5vw-3h65-2q3v",
                     "aliases": [],
                     "weighted_severity": null,
                     "exploitability_score": null,
@@ -189,6 +190,7 @@ Example response:
                 "affected_by_vulnerabilities": [
                 {
                     "advisory_id": "GHSA-g5vw-3h65-2q3v",
+                    "advisory_uid": "ghsa/g5vw-3h65-2q3v",
                     "aliases": [],
                     "weighted_severity": null,
                     "exploitability_score": null,

vulnerabilities/api_v3.py

@@ -113,7 +113,7 @@ class AdvisoryV3Serializer(serializers.ModelSerializer):
     weaknesses = AdvisoryWeaknessSerializer(many=True)
     references = AdvisoryReferenceSerializer(many=True)
     severities = AdvisorySeveritySerializer(many=True)
-    advisory_id = serializers.CharField(source="avid", read_only=True)
+    advisory_uid = serializers.CharField(source="avid", read_only=True)
     related_ssvc_trees = serializers.SerializerMethodField()
 
     def get_related_ssvc_trees(self, obj):
@@ -143,6 +143,7 @@ class Meta:
         model = AdvisoryV2
         fields = [
             "advisory_id",
+            "advisory_uid",
             "url",
             "aliases",
             "summary",
@@ -270,6 +271,7 @@ def get_affected_by_vulnerabilities(self, package):
                 result.append(
                     {
                         "advisory_id": advisory.advisory_id.split("/")[-1],
+                        "advisory_uid": advisory.avid,
                         "aliases": [alias.alias for alias in advisory.aliases.all()],
                         "summary": advisory.summary,
                         "severity": advisory.weighted_severity,
@@ -313,6 +315,7 @@ def get_fixing_vulnerabilities(self, package):
                 results.append(
                     {
                         "advisory_id": advisory.advisory_id.split("/")[-1],
+                        "advisory_uid": advisory.avid,
                     }
                 )
             return results
@@ -337,6 +340,7 @@ def return_fixing_advisories_data(self, advisories):
             result.append(
                 {
                     "advisory_id": advisory.identifier,
+                    "advisory_uid": advisory.advisory.avid,
                 }
             )
 
@@ -364,6 +368,7 @@ def return_advisories_data(self, package, advisories_qs, advisories):
             result.append(
                 {
                     "advisory_id": advisory.identifier,
+                    "advisory_uid": advisory.advisory.avid,
                     "aliases": [alias.alias for alias in advisory.aliases],
                     "weighted_severity": advisory.weighted_severity,
                     "exploitability": advisory.exploitability,
@@ -471,6 +476,7 @@ def create(self, request, *args, **kwargs):
 
 class AffectedByAdvisoryV3Serializer(AdvisoryV3Serializer):
     fixed_by_packages = serializers.SerializerMethodField()
+    advisory_uid = serializers.CharField(source="avid", read_only=True)
 
     def get_fixed_by_packages(self, obj):
         return list(
@@ -483,6 +489,7 @@ class Meta:
         model = AdvisoryV2
         fields = [
             "advisory_id",
+            "advisory_uid",
             "url",
             "aliases",
             "summary",
@@ -623,8 +630,8 @@ def get_affected_advisories_bulk(packages):
 
             grouped.append(
                 {
-                    "avid": primary.avid,
                     "advisory_id": identifier,
+                    "advisory_uid": primary.avid,
                     "aliases": aliases,
                     "weighted_severity": weighted_severity,
                     "exploitability": exploitability,
@@ -699,7 +706,9 @@ def get_fixing_advisories_bulk(packages):
         grouped = []
 
         for adv_id in groups:
-            grouped.append({"advisory_id": adv_id.split("/")[-1]})
+            grouped.append(
+                {"advisory_id": adv_id.split("/")[-1], "advisory_uid": adv_id.split("/")[-1]}
+            )
 
         result[package.id] = grouped
 

vulnerabilities/migrations/0130_advisoryv2_pipeline_id.py

@@ -0,0 +1,24 @@
+# Generated by Django 5.2.11 on 2026-05-18 08:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0129_advisorypoc"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                blank=True,
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=200,
+                null=True,
+            ),
+        ),
+    ]

vulnerabilities/migrations/0131_auto_20260518_0854.py

@@ -0,0 +1,42 @@
+# Generated by Django 5.2.11 on 2026-05-18 08:54
+
+from django.db import migrations, models
+from django.db.models import F
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0130_advisoryv2_pipeline_id"),
+    ]
+
+    def populate_pipeline_id(apps, schema_editor):
+        Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+        Advisory.objects.update(
+            pipeline_id=F("datasource_id")
+        )
+
+        assert not Advisory.objects.filter(pipeline_id="").exists(), "Some advisories have an empty pipeline_id after the update"
+
+    operations = [
+        migrations.RunPython(populate_pipeline_id, reverse_code=migrations.RunPython.noop),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="pipeline_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the pipeline used for this advisory .e.g.: nginx_importer_v2",
+                max_length=200,
+            ),
+        ),
+        migrations.AlterField(
+            model_name="advisoryv2",
+            name="datasource_id",
+            field=models.CharField(
+                db_index=True,
+                help_text="Unique ID for the datasource used for this advisory .e.g.: nginx",
+                max_length=200,
+            ),
+        ),
+    ]

vulnerabilities/migrations/0132_migrate_advisoryv2_datasource_ids.py

@@ -0,0 +1,77 @@
+from django.db import migrations
+from django.db.models import F
+from django.db.models import Value
+from django.db.models.functions import Concat
+
+
+def migrate_advisoryv2_datasource_ids(apps, schema_editor):
+    """
+    v2 importers previously stored pipeline_id as datasource_id on AdvisoryV2.
+    Migration 0131 copied that value into pipeline_id; update datasource_id (and avid)
+    to each pipeline's datasource_id, matching rows by pipeline_id.
+    """
+    from vulnerabilities.importers import IMPORTERS_REGISTRY
+    from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+
+    Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+    for pipeline_class in IMPORTERS_REGISTRY.values():
+        if not issubclass(pipeline_class, VulnerableCodeBaseImporterPipelineV2):
+            continue
+
+        pipeline_id = pipeline_class.pipeline_id
+        datasource_id = pipeline_class.datasource_id
+        if not pipeline_id or not datasource_id:
+            continue
+        if pipeline_id == datasource_id:
+            continue
+
+        Advisory.objects.filter(
+            pipeline_id=pipeline_id,
+            datasource_id=pipeline_id,
+        ).update(
+            datasource_id=datasource_id,
+            avid=Concat(Value(f"{datasource_id}/"), F("advisory_id")),
+        )
+
+
+def reverse_migrate_advisoryv2_datasource_ids(apps, schema_editor):
+    from vulnerabilities.importers import IMPORTERS_REGISTRY
+    from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
+
+    Advisory = apps.get_model("vulnerabilities", "AdvisoryV2")
+
+    for pipeline_class in IMPORTERS_REGISTRY.values():
+        if not issubclass(pipeline_class, VulnerableCodeBaseImporterPipelineV2):
+            continue
+        if "v2_importers" not in pipeline_class.__module__:
+            continue
+
+        pipeline_id = pipeline_class.pipeline_id
+        datasource_id = pipeline_class.datasource_id
+        if not pipeline_id or not datasource_id:
+            continue
+        if pipeline_id == datasource_id:
+            continue
+
+        Advisory.objects.filter(
+            pipeline_id=pipeline_id,
+            datasource_id=datasource_id,
+        ).update(
+            datasource_id=pipeline_id,
+            avid=Concat(Value(f"{pipeline_id}/"), F("advisory_id")),
+        )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("vulnerabilities", "0131_auto_20260518_0854"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            migrate_advisoryv2_datasource_ids,
+            reverse_code=reverse_migrate_advisoryv2_datasource_ids,
+        ),
+    ]

vulnerabilities/models.py

@@ -3058,7 +3058,15 @@ class AdvisoryV2(models.Model):
         blank=False,
         null=False,
         db_index=True,
-        help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx_importer_v2",
+        help_text="Unique ID for the datasource used for this advisory ." "e.g.: nginx",
+    )
+
+    pipeline_id = models.CharField(
+        max_length=200,
+        blank=False,
+        null=False,
+        db_index=True,
+        help_text="Unique ID for the pipeline used for this advisory ." "e.g.: nginx_importer_v2",
     )
 
     # This is similar to a name

vulnerabilities/pipelines/__init__.py

@@ -273,7 +273,7 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
 
     pipeline_id = None  # Unique Pipeline ID, this should be the name of pipeline module.
     license_url = None
-    datasource_name = None
+    datasource_id = None
     spdx_license_expression = None
     repo_url = None
     ignorable_versions = []
@@ -319,6 +319,9 @@ def advisories_count(self) -> int:
         raise NotImplementedError
 
     def collect_and_store_advisories(self):
+        if not self.pipeline_id and not self.datasource_id:
+            self.log("Pipeline must have a unique pipeline_id or datasource_id defined.")
+            return
         collected_advisory_count = 0
         estimated_advisory_count = self.advisories_count()
 
@@ -338,6 +341,7 @@ def collect_and_store_advisories(self):
                 if _obj := insert_advisory_v2(
                     advisory=advisory,
                     pipeline_id=self.pipeline_id,
+                    datasource_id=self.datasource_id,
                     logger=self.log,
                     precedence=self.precedence,
                 ):

vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py

@@ -38,6 +38,7 @@ class AlpineLinuxImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
 
     pipeline_id = "alpine_linux_importer_v2"
     spdx_license_expression = "CC-BY-SA-4.0"
+    datasource_id = "alpine_linux"
     license_url = "https://secdb.alpinelinux.org/license.txt"
     repo_url = "git+https://github.com/aboutcode-org/aboutcode-mirror-alpine-secdb/"