feat(finding): pluggable false-positive-history candidate filter

docs/content/releases/os_upgrading/3.2.md

@@ -0,0 +1,58 @@
+---
+title: 'Upgrading to DefectDojo Version 3.2.x'
+toc_hide: true
+weight: -20260701
+description: Vulnerability ids gain an autodetected type and a uniqueness constraint; findings can now carry multiple CWEs via a new Finding_CWE relationship. Migrations add the type column, de-duplicate vulnerability-id rows, add the uniqueness constraint, create the CWE table, and backfill it. Existing hash codes are unaffected.
+---
+
+## Vulnerability id type
+
+Each `Vulnerability_Id` gains an autodetected `vulnerability_id_type` — the identifier's leading
+prefix (`CVE-2024-1234` → `CVE`, `GHSA-…` → `GHSA`, `RUSTSEC-…` → `RUSTSEC`). It is derived
+structurally (no registry) and stored (indexed) so identifiers can be filtered and grouped by type
+efficiently. It is `NULL` when there is no non-numeric prefix. It is populated automatically on
+import and on `save()`; existing rows are backfilled by migration. It does not participate in
+`hash_code`, so **existing hash codes and deduplication are unaffected**.
+
+A unique constraint is also added on `(finding, vulnerability_id)`; pre-existing duplicate rows
+(unintended) are consolidated first.
+
+## Multiple CWEs per finding
+
+A finding could previously store only one CWE (the integer `cwe` field). This release adds a
+dedicated `Finding_CWE` relationship so a finding can carry **multiple CWEs**, using the same
+approach as vulnerability ids: the primary CWE stays on `Finding.cwe` (unchanged — legacy
+deduplication and hash codes still use it), and additional CWEs live in the relationship.
+
+CWE is modeled separately from vulnerability identifiers on purpose: a CWE is a weakness *class*,
+not a vulnerability *instance* identifier, so it must not participate in `hash_code`,
+vulnerability-id deduplication, or the `cve` field. Because of this separation, **existing hash
+codes and deduplication are unaffected**.
+
+CWEs are populated automatically on import and when a finding is created or edited (from the
+finding's CWE field, plus any additional CWEs a parser supplies). The finding exposes them via
+`finding.cwes` (primary first, deduplicated).
+
+## Database migration
+
+Five migrations run automatically on upgrade:
+
+- `0276_vulnerability_id_type` — adds the indexed `vulnerability_id_type` column and a leading
+  index on `vulnerability_id`.
+- `0277_backfill_vulnerability_id_type` — backfills `vulnerability_id_type` and removes duplicate
+  `(finding, vulnerability_id)` rows (keeping the earliest).
+- `0278_unique_finding_vulnerability_id` — adds the unique constraint on `(finding, vulnerability_id)`.
+- `0279_finding_cwe` — creates the `Finding_CWE` table (unique per `(finding, cwe)`).
+- `0280_backfill_finding_cwe` — seeds `Finding_CWE` rows from the legacy `Finding.cwe` values.
+
+### What you need to do
+
+The migrations are applied automatically. New and edited findings populate their CWE relationship
+automatically. To backfill `Finding_CWE` rows for **existing** findings, run the idempotent command
+after upgrading:
+
+```
+manage.py migrate_cwe
+```
+
+For more information, check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/3.2.0).

dojo/db_migrations/0276_vulnerability_id_type.py

@@ -0,0 +1,25 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    """Schema only: add the autodetected vulnerability_id_type column and a leading index on
+    vulnerability_id. The data backfill (0277) and the unique constraint (0278, which requires
+    the dedupe data step first) are kept in separate migrations so data migrations are never
+    mixed with schema migrations."""
+
+    dependencies = [
+        ("dojo", "0275_usercontactinfo_user_state_details"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="vulnerability_id",
+            name="vulnerability_id_type",
+            field=models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True),
+        ),
+        migrations.AddIndex(
+            model_name="vulnerability_id",
+            index=models.Index(fields=["vulnerability_id"], name="dojo_vuln_id_lookup_idx"),
+        ),
+    ]

dojo/db_migrations/0277_backfill_vulnerability_id_type.py

@@ -0,0 +1,49 @@
+from django.db import migrations
+
+from dojo.finding.vulnerability_id import resolve_vulnerability_id_type
+
+BATCH_SIZE = 1000
+
+
+def dedupe_vulnerability_ids(apps, schema_editor):
+    """Remove duplicate (finding, vulnerability_id) rows, keeping the lowest id, so the unique
+    constraint added in the following schema migration (0278) can be created. Postgres."""
+    schema_editor.execute(
+        """
+        DELETE FROM dojo_vulnerability_id a
+        USING dojo_vulnerability_id b
+        WHERE a.finding_id = b.finding_id
+          AND a.vulnerability_id = b.vulnerability_id
+          AND a.id > b.id
+        """,
+    )
+
+
+def backfill_vulnerability_id_type(apps, schema_editor):
+    """Populate vulnerability_id_type for existing rows by autodetecting the id prefix."""
+    Vulnerability_Id = apps.get_model("dojo", "Vulnerability_Id")
+    batch = []
+    qs = Vulnerability_Id.objects.filter(vulnerability_id_type__isnull=True).only("id", "vulnerability_id")
+    for row in qs.iterator(chunk_size=BATCH_SIZE):
+        row.vulnerability_id_type = resolve_vulnerability_id_type(row.vulnerability_id)
+        batch.append(row)
+        if len(batch) >= BATCH_SIZE:
+            Vulnerability_Id.objects.bulk_update(batch, ["vulnerability_id_type"], batch_size=BATCH_SIZE)
+            batch = []
+    if batch:
+        Vulnerability_Id.objects.bulk_update(batch, ["vulnerability_id_type"], batch_size=BATCH_SIZE)
+
+
+class Migration(migrations.Migration):
+
+    """Data only (no schema changes): dedupe (finding, vulnerability_id) rows and backfill
+    vulnerability_id_type for existing rows."""
+
+    dependencies = [
+        ("dojo", "0276_vulnerability_id_type"),
+    ]
+
+    operations = [
+        migrations.RunPython(dedupe_vulnerability_ids, migrations.RunPython.noop),
+        migrations.RunPython(backfill_vulnerability_id_type, migrations.RunPython.noop),
+    ]

dojo/db_migrations/0278_unique_finding_vulnerability_id.py

@@ -0,0 +1,18 @@
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    """Schema only: add the (finding, vulnerability_id) unique constraint. Runs after the 0277
+    dedupe data step so the constraint can be created without violating existing duplicate rows."""
+
+    dependencies = [
+        ("dojo", "0277_backfill_vulnerability_id_type"),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name="vulnerability_id",
+            constraint=models.UniqueConstraint(fields=("finding", "vulnerability_id"), name="unique_finding_vulnerability_id"),
+        ),
+    ]

dojo/db_migrations/0279_finding_cwe.py

@@ -0,0 +1,28 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    """Schema only: create the Finding_CWE relationship (multiple CWEs per finding). The data
+    backfill from the legacy Finding.cwe field is kept in a separate migration (0280) so data
+    migrations are never mixed with schema migrations."""
+
+    dependencies = [
+        ("dojo", "0278_unique_finding_vulnerability_id"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Finding_CWE",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("cwe", models.CharField(db_index=True, max_length=11)),
+                ("finding", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to="dojo.finding")),
+            ],
+        ),
+        migrations.AddConstraint(
+            model_name="finding_cwe",
+            constraint=models.UniqueConstraint(fields=("finding", "cwe"), name="unique_finding_cwe"),
+        ),
+    ]

dojo/db_migrations/0280_backfill_finding_cwe.py

@@ -0,0 +1,36 @@
+from django.db import migrations
+
+from dojo.finding.cwe import cwe_label
+
+BATCH_SIZE = 1000
+
+
+def create_finding_cwe_records(apps, schema_editor):
+    """Backfill Finding_CWE rows (canonical CWE-<n>) from the legacy int Finding.cwe field."""
+    Finding = apps.get_model("dojo", "Finding")
+    Finding_CWE = apps.get_model("dojo", "Finding_CWE")
+    batch = []
+    for finding in Finding.objects.filter(cwe__gt=0).only("id", "cwe").iterator(chunk_size=BATCH_SIZE):
+        label = cwe_label(finding.cwe)
+        if label is None:
+            continue
+        batch.append(Finding_CWE(finding_id=finding.id, cwe=label))
+        if len(batch) >= BATCH_SIZE:
+            Finding_CWE.objects.bulk_create(batch, batch_size=BATCH_SIZE, ignore_conflicts=True)
+            batch = []
+    if batch:
+        Finding_CWE.objects.bulk_create(batch, batch_size=BATCH_SIZE, ignore_conflicts=True)
+
+
+class Migration(migrations.Migration):
+
+    """Data only (no schema changes): create the initial Finding_CWE rows from the legacy
+    Finding.cwe field."""
+
+    dependencies = [
+        ("dojo", "0279_finding_cwe"),
+    ]
+
+    operations = [
+        migrations.RunPython(create_finding_cwe_records, migrations.RunPython.noop),
+    ]

dojo/finding/api/serializer.py

@@ -15,7 +15,9 @@
 import dojo.finding.helper as finding_helper
 from dojo.authorization.authorization import user_has_permission
 from dojo.celery_dispatch import dojo_dispatch_task
+from dojo.finding.cwe import cwe_label, cwe_number
 from dojo.finding.helper import (
+    save_cwes,
     save_endpoints_template,
     save_vulnerability_ids,
     save_vulnerability_ids_template,
@@ -32,6 +34,7 @@
     Endpoint,
     Engagement,
     Finding,
+    Finding_CWE,
     Finding_Group,
     Finding_Template,
     Note_Type,
@@ -298,6 +301,30 @@ class Meta:
         fields = ["vulnerability_id"]
 
 
+@extend_schema_field(serializers.CharField())
+class CweField(serializers.Field):
+
+    """Serialize a CWE as the canonical ``CWE-<n>`` string; accept ``"CWE-79"`` or ``"79"`` on write."""
+
+    def to_representation(self, value):
+        return cwe_label(value) or value
+
+    def to_internal_value(self, data):
+        label = cwe_label(data)
+        if label is None:
+            msg = "Enter a CWE number, e.g. 89 or CWE-89."
+            raise serializers.ValidationError(msg)
+        return label
+
+
+class FindingCweSerializer(serializers.ModelSerializer):
+    cwe = CweField()
+
+    class Meta:
+        model = Finding_CWE
+        fields = ["cwe"]
+
+
 class FindingSerializer(serializers.ModelSerializer):
     mitigated = serializers.DateTimeField(required=False, allow_null=True)
     mitigated_by = serializers.PrimaryKeyRelatedField(required=False, allow_null=True, queryset=User.objects.all())
@@ -321,6 +348,9 @@ class FindingSerializer(serializers.ModelSerializer):
     vulnerability_ids = VulnerabilityIdSerializer(
         source="vulnerability_id_set", many=True, required=False,
     )
+    cwes = FindingCweSerializer(
+        source="finding_cwe_set", many=True, required=False,
+    )
     reporter = serializers.PrimaryKeyRelatedField(
         required=False, queryset=User.objects.all(),
     )
@@ -417,6 +447,13 @@ def update(self, instance, validated_data):
             logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0])
             validated_data["cve"] = parsed_vulnerability_ids[0]
 
+        # CWEs (mirror vulnerability_ids): the first entry is the primary Finding.cwe; the rest
+        # become Finding_CWE rows via save_cwes() below.
+        parsed_cwes = None
+        if (cwes := validated_data.pop("finding_cwe_set", None)) is not None:
+            parsed_cwes = [entry["cwe"] for entry in cwes]
+            validated_data["cwe"] = cwe_number(parsed_cwes[0]) if parsed_cwes else 0
+
         # Save the reporter on the finding
         if reporter_id := validated_data.get("reporter"):
             instance.reporter = reporter_id
@@ -445,6 +482,11 @@ def update(self, instance, validated_data):
             instance, validated_data,
         )
 
+        # Sync the CWE relation (separate from vulnerability ids) after the new cwe is applied.
+        if parsed_cwes is not None:
+            instance.unsaved_cwes = parsed_cwes[1:]
+        save_cwes(instance)
+
         if settings.V3_FEATURE_LOCATIONS and locations is not None:
             for location_ref in instance.locations.all():
                 location_ref.location.disassociate_from_finding(instance)
@@ -561,6 +603,9 @@ class FindingCreateSerializer(serializers.ModelSerializer):
     vulnerability_ids = VulnerabilityIdSerializer(
         source="vulnerability_id_set", many=True, required=False,
     )
+    cwes = FindingCweSerializer(
+        source="finding_cwe_set", many=True, required=False,
+    )
     reporter = serializers.PrimaryKeyRelatedField(
         required=False, queryset=User.objects.all(),
     )
@@ -601,6 +646,12 @@ def create(self, validated_data):
             validated_data["cve"] = parsed_vulnerability_ids[0]
             # validated_data["unsaved_vulnerability_ids"] = parsed_vulnerability_ids
 
+        # CWEs (mirror vulnerability_ids): first entry is the primary cwe, the rest are extras.
+        parsed_cwes = None
+        if (cwes := validated_data.pop("finding_cwe_set", None)) is not None:
+            parsed_cwes = [entry["cwe"] for entry in cwes]
+            validated_data["cwe"] = cwe_number(parsed_cwes[0]) if parsed_cwes else 0
+
         # super.create() doesn't accept unsaved_vulnerability_ids or dedupe_option=False, so call save directly.
         new_finding = Finding(**validated_data)
         new_finding.unsaved_vulnerability_ids = parsed_vulnerability_ids or []
@@ -617,6 +668,9 @@ def create(self, validated_data):
             new_finding.reviewers.set(reviewers)
         if parsed_vulnerability_ids:
             save_vulnerability_ids(new_finding, parsed_vulnerability_ids)
+        if parsed_cwes is not None:
+            new_finding.unsaved_cwes = parsed_cwes[1:]
+        save_cwes(new_finding)
 
         if push_to_jira:
             jira_services.push(new_finding)

dojo/finding/cwe.py

@@ -0,0 +1,49 @@
+"""Pure helpers for CWE identifiers (no model imports, safe to import anywhere)."""
+
+
+def cwe_number(value) -> int | None:
+    """
+    ``"CWE-79"`` / ``"79"`` / ``79`` -> ``79`` (positive int, case-insensitive); anything else -> ``None``.
+
+    CWE numbers are positive, so ``0`` (Finding.cwe's "unset" sentinel) and non-numeric input -> ``None``.
+    """
+    if value is None:
+        return None
+    token = str(value).strip().upper().removeprefix("CWE-")
+    if not token.isdigit():
+        return None
+    number = int(token)
+    return number if number > 0 else None
+
+
+def cwe_label(value) -> str | None:
+    """``79`` / ``"79"`` / ``"CWE-79"`` -> ``"CWE-79"`` (canonical, case-insensitive); invalid -> ``None``."""
+    number = cwe_number(value)
+    return f"CWE-{number}" if number is not None else None
+
+
+def parse_cwes(text: str | None) -> list[str]:
+    """
+    Parse CWEs from user text (one per line or comma-separated) into canonical ``CWE-<n>`` labels.
+
+    Accepts ``89`` or ``CWE-89`` (case-insensitive); ignores anything non-numeric; deduplicates.
+    """
+    result: list[str] = []
+    for token in (text or "").replace(",", "\n").split():
+        label = cwe_label(token)
+        if label is not None and label not in result:
+            result.append(label)
+    return result
+
+
+def finding_cwe_labels(cwe, unsaved_cwes=None) -> list[str]:
+    """Canonical ``CWE-<n>`` labels for a finding: the primary ``cwe`` first, then extras, deduplicated."""
+    labels: list[str] = []
+    primary = cwe_label(cwe)
+    if primary is not None:
+        labels.append(primary)
+    for extra in unsaved_cwes or []:
+        label = cwe_label(extra)
+        if label is not None:
+            labels.append(label)
+    return list(dict.fromkeys(labels))