From 577803a126d251159a8be063c9aa37dd47d3afee Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 10:16:25 -0600 Subject: [PATCH 01/18] feat(vuln-id): VulnerabilityId entity + FindingVulnerabilityReference models Introduce the OSS vulnerability_id module (schema only, no behavior change): - dojo/vulnerability_id/models.py: VulnerabilityId (global registry of id strings, EPSS/KEV columns nullable, FTS/trigram/Upper indexes) and FindingVulnerabilityReference (ordered Finding->VulnerabilityId link, order==0 is primary). Plain models.Model with explicit created/updated: BaseModel's manager would force order_by("id") and override the Meta.ordering=["order"] the reference contract depends on, and its full_clean-on-save must stay out of bulk paths. - dojo/vulnerability_id/admin.py + __init__ admin import (mirrors dojo/finding, dojo/location; autodiscover only scans top-level dojo.admin). - dojo/vulnerability_id/queries.py: get_auth_filter-delegating seam accessors. - dojo/models.py: facade re-export (string FKs => no circular import). - settings.dist.py: DD_V3_FEATURE_VULNERABILITY_IDS=(bool, True) + assignment. - prefetch/registrations.py + authorization/query_registrations.py: register both models (unregistered => dropped from prefetch) with RBAC filters under vulnerability_id.get_authorized_entities / .get_authorized_references. makemigrations --check reports exactly these two models pending; manage.py check clean; ruff clean. Co-Authored-By: Claude Fable 5 --- dojo/api_v2/prefetch/registrations.py | 10 ++++ dojo/authorization/query_registrations.py | 28 +++++++++ dojo/models.py | 4 ++ dojo/settings/settings.dist.py | 5 ++ dojo/vulnerability_id/__init__.py | 1 + dojo/vulnerability_id/admin.py | 15 +++++ dojo/vulnerability_id/models.py | 71 +++++++++++++++++++++++ dojo/vulnerability_id/queries.py | 24 ++++++++ 8 files changed, 158 insertions(+) create mode 100644 dojo/vulnerability_id/__init__.py create mode 100644 dojo/vulnerability_id/admin.py create mode 100644 dojo/vulnerability_id/models.py create mode 100644 dojo/vulnerability_id/queries.py diff --git a/dojo/api_v2/prefetch/registrations.py b/dojo/api_v2/prefetch/registrations.py index 44ecdc62bb7..e9922086471 100644 --- a/dojo/api_v2/prefetch/registrations.py +++ b/dojo/api_v2/prefetch/registrations.py @@ -110,6 +110,14 @@ from dojo.test.queries import get_authorized_test_imports, get_authorized_tests from dojo.tool_product.queries import get_authorized_tool_product_settings from dojo.url.models import URL +from dojo.vulnerability_id.models import ( + FindingVulnerabilityReference, + VulnerabilityId, +) +from dojo.vulnerability_id.queries import ( + get_authorized_finding_vulnerability_references, + get_authorized_vulnerability_id_entities, +) ######## # Models backed by ViewSets (api_v2.views) from which we can derive the required permission check. @@ -216,6 +224,8 @@ for model, helper in ( (Finding_Group, get_authorized_finding_groups), (Vulnerability_Id, get_authorized_vulnerability_ids), + (VulnerabilityId, get_authorized_vulnerability_id_entities), + (FindingVulnerabilityReference, get_authorized_finding_vulnerability_references), ): register(model, discard_user(helper), "view") diff --git a/dojo/authorization/query_registrations.py b/dojo/authorization/query_registrations.py index 0e5f26a5c05..b808dc50092 100644 --- a/dojo/authorization/query_registrations.py +++ b/dojo/authorization/query_registrations.py @@ -41,6 +41,7 @@ Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task +from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId def _resolve_user(user): @@ -418,6 +419,33 @@ def _get_authorized_vulnerability_ids(permission, queryset=None, user=None): register_auth_filter("finding.get_authorized_vulnerability_ids_for_queryset", _get_authorized_vulnerability_ids) +def _get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): + user = _resolve_user(user) + qs = queryset if queryset is not None else VulnerabilityId.objects.all() + if user is None or getattr(user, "is_anonymous", False): + return qs.none() + if _is_unrestricted(user, permission_to_action(permission)): + return qs + # An entity links to findings across many products; distinct() collapses the join fan-out. + return qs.filter( + finding_references__finding__test__engagement__product__id__in=_authorized_product_ids(user), + ).distinct() + + +def _get_authorized_finding_vulnerability_references(permission, queryset=None, user=None): + user = _resolve_user(user) + qs = queryset if queryset is not None else FindingVulnerabilityReference.objects.all() + if user is None or getattr(user, "is_anonymous", False): + return qs.none() + if _is_unrestricted(user, permission_to_action(permission)): + return qs + return qs.filter(finding__test__engagement__product__id__in=_authorized_product_ids(user)) + + +register_auth_filter("vulnerability_id.get_authorized_entities", _get_authorized_vulnerability_id_entities) +register_auth_filter("vulnerability_id.get_authorized_references", _get_authorized_finding_vulnerability_references) + + # --------------------------------------------------------------------------- # User queries # --------------------------------------------------------------------------- diff --git a/dojo/models.py b/dojo/models.py index 1d38c951bb7..bd3f4ac3386 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -407,6 +407,10 @@ class Meta: Finding_Template, Vulnerability_Id, # noqa: F401 -- re-export ) +from dojo.vulnerability_id.models import ( # noqa: E402 -- re-export; FKs reference dojo.Finding / dojo.VulnerabilityId by string + FindingVulnerabilityReference, # noqa: F401 -- re-export + VulnerabilityId, # noqa: F401 -- re-export +) class Check_List(models.Model): diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index b590576a62e..f494cdd928d 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -288,6 +288,10 @@ DD_REQUESTS_TIMEOUT=(int, 30), # Dictates if v3 functionality will be enabled (on by default as of 3.0.0; set to False to revert to the legacy Endpoint model) DD_V3_FEATURE_LOCATIONS=(bool, True), + # Dictates if findings read their vulnerability ids from the normalized VulnerabilityId entity/reference tables + # (on by default; set to False to read from the legacy Vulnerability_Id table). Writes are always dual, so the flag + # only selects the READ store and is reversible in both directions with zero data drift. + DD_V3_FEATURE_VULNERABILITY_IDS=(bool, True), # Dictates if v3 org/asset relabeling (+url routing) will be enabled (on by default as of 3.0.0; set to False to restore Product/Product Type labels and URLs) DD_ENABLE_V3_ORGANIZATION_ASSET_RELABEL=(bool, True), # Shared cache backend (django.core.cache). When set, Django uses RedisCache @@ -679,6 +683,7 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # V3 Feature Flags V3_FEATURE_LOCATIONS = env("DD_V3_FEATURE_LOCATIONS") +V3_FEATURE_VULNERABILITY_IDS = env("DD_V3_FEATURE_VULNERABILITY_IDS") # ------------------------------------------------------------------------------ diff --git a/dojo/vulnerability_id/__init__.py b/dojo/vulnerability_id/__init__.py new file mode 100644 index 00000000000..27909fcd3bf --- /dev/null +++ b/dojo/vulnerability_id/__init__.py @@ -0,0 +1 @@ +import dojo.vulnerability_id.admin # noqa: F401 diff --git a/dojo/vulnerability_id/admin.py b/dojo/vulnerability_id/admin.py new file mode 100644 index 00000000000..1f44929c958 --- /dev/null +++ b/dojo/vulnerability_id/admin.py @@ -0,0 +1,15 @@ +from django.contrib import admin + +from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId + + +@admin.register(VulnerabilityId) +class VulnerabilityIdAdmin(admin.ModelAdmin): + + """Admin support for the VulnerabilityId entity model.""" + + +@admin.register(FindingVulnerabilityReference) +class FindingVulnerabilityReferenceAdmin(admin.ModelAdmin): + + """Admin support for the FindingVulnerabilityReference model.""" diff --git a/dojo/vulnerability_id/models.py b/dojo/vulnerability_id/models.py new file mode 100644 index 00000000000..84e58b28baa --- /dev/null +++ b/dojo/vulnerability_id/models.py @@ -0,0 +1,71 @@ +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector +from django.core.validators import MaxValueValidator, MinValueValidator +from django.db import models +from django.db.models.functions import Upper + + +class VulnerabilityId(models.Model): + + """ + Global registry of vulnerability identifier strings (CVE-..., GHSA-..., ...). + + Stores each identifier EXACTLY as supplied (never case-normalized): the string is + simultaneously the natural key, the display form, and the hash_code input, so any + normalization would change Finding.get_vulnerability_ids() output and churn hashes. + Case-insensitive matching is an index concern (see Upper index), not storage. + Rows are never deleted by finding write paths — orphan entities are the (future) + enrichment registry. EPSS/KEV columns are NULL until enrichment populates them + (NULL = "not enriched", distinct from False). + """ + + vulnerability_id = models.TextField(unique=True, blank=False, null=False, + verbose_name="Vulnerability Id") + # Autodetected prefix (CVE, GHSA, ...) via dojo.finding.vulnerability_id. + # resolve_vulnerability_id_type; stamped at entity creation. Excluded from hashing. + vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, + editable=False, db_index=True) + epss_score = models.FloatField(null=True, blank=True, default=None, + validators=[MinValueValidator(0.0), MaxValueValidator(1.0)]) + epss_percentile = models.FloatField(null=True, blank=True, default=None, + validators=[MinValueValidator(0.0), MaxValueValidator(1.0)]) + known_exploited = models.BooleanField(null=True, blank=True, default=None) + ransomware_used = models.BooleanField(null=True, blank=True, default=None) + kev_date = models.DateField(null=True, blank=True, default=None) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + GinIndex(SearchVector("vulnerability_id", weight="A", config="english"), + name="dojo_vulnid_entity_fts_gin"), + GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], + name="dojo_vulnid_entity_trgm"), + models.Index(Upper("vulnerability_id"), name="dojo_vulnid_entity_upper"), + ] + + def __str__(self): + return self.vulnerability_id + + +class FindingVulnerabilityReference(models.Model): + + """Ordered link Finding -> VulnerabilityId. order == 0 is the primary id (== Finding.cve).""" + + finding = models.ForeignKey("dojo.Finding", on_delete=models.CASCADE, + related_name="vulnerability_references", editable=False) + vulnerability = models.ForeignKey("dojo.VulnerabilityId", on_delete=models.CASCADE, + related_name="finding_references", editable=False) + order = models.PositiveSmallIntegerField(default=0, + help_text="Position in the finding's id list; 0 is the primary identifier.") + created = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["order"] + constraints = [models.UniqueConstraint(fields=["finding", "vulnerability"], + name="unique_finding_vulnerability")] + indexes = [models.Index(fields=["finding", "order"]), + models.Index(fields=["vulnerability"])] + + def __str__(self): + return self.vulnerability.vulnerability_id diff --git a/dojo/vulnerability_id/queries.py b/dojo/vulnerability_id/queries.py new file mode 100644 index 00000000000..5fc97d7eede --- /dev/null +++ b/dojo/vulnerability_id/queries.py @@ -0,0 +1,24 @@ +import logging + +try: + from dojo.authorization.query_filters import get_auth_filter +except ImportError: + def get_auth_filter(key): return None + +from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId + +logger = logging.getLogger(__name__) + + +def get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): + impl = get_auth_filter("vulnerability_id.get_authorized_entities") + if impl: + return impl(permission, queryset=queryset, user=user) + return VulnerabilityId.objects.all().order_by("id") if queryset is None else queryset + + +def get_authorized_finding_vulnerability_references(permission, queryset=None, user=None): + impl = get_auth_filter("vulnerability_id.get_authorized_references") + if impl: + return impl(permission, queryset=queryset, user=user) + return FindingVulnerabilityReference.objects.all().order_by("id") if queryset is None else queryset From 8495401f5a0c091968482fc223020dca746ece81 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:19:58 -0600 Subject: [PATCH 02/18] feat(vuln-id): entity tables + idempotent set-based backfill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 0285_vulnerability_id_entity_tables: CreateModel x2 (empty tables => index DDL instant; >50M-row GIN-strip fallback documented in the header). - 0286_backfill_vulnerability_id_entities: atomic=False, Postgres-guarded, RunPython forward -> dojo.vulnerability_id.backfill.run_backfill (noop reverse). - backfill.run_backfill: idempotent, insert-only, set-based SQL shared with the WP5 migrate_vulnerability_ids command — (1) entities from legacy table (MAX type carry), (2) cve-only entities + Python type-stamp, (3) references windowed by finding_id (ROW_NUMBER cve-first then legacy-PK; order 0-based), (4) order-0 refs for cve-only findings. ON CONFLICT DO NOTHING everywhere. - test_migrations.TestVulnerabilityIdBackfill: seeds ordered / cve-drift / case-variant / cve-only / null-cve findings, asserts entity extraction, type, per-finding order (cve at 0), pair parity, and idempotent re-run. Uses a plain TestCase against run_backfill rather than MigratorTestCase: django_test_migrations replays the full squashed chain from a wiped schema in setUp and collides on legacy tables (dojo_cred_user) — the same reason the only other MigratorTestCase in this file is skipped. Migration application itself is covered by the create-db and dev-DB migrate runs. Verified: 0285/0286 apply cleanly on a fresh create-db build and on the dev DB; backfill test green; re-running run_backfill is a no-op. Co-Authored-By: Claude Fable 5 --- .../0285_vulnerability_id_entity_tables.py | 60 +++++++ ...0286_backfill_vulnerability_id_entities.py | 42 +++++ dojo/vulnerability_id/backfill.py | 140 ++++++++++++++++ unittests/test_migrations.py | 158 ++++++++++++++++++ 4 files changed, 400 insertions(+) create mode 100644 dojo/db_migrations/0285_vulnerability_id_entity_tables.py create mode 100644 dojo/db_migrations/0286_backfill_vulnerability_id_entities.py create mode 100644 dojo/vulnerability_id/backfill.py diff --git a/dojo/db_migrations/0285_vulnerability_id_entity_tables.py b/dojo/db_migrations/0285_vulnerability_id_entity_tables.py new file mode 100644 index 00000000000..43707fd979d --- /dev/null +++ b/dojo/db_migrations/0285_vulnerability_id_entity_tables.py @@ -0,0 +1,60 @@ +# Vulnerability ID entity tables: the normalized VulnerabilityId registry and the ordered +# Finding -> VulnerabilityId reference table. Both tables are CREATED EMPTY here, so every index +# (including the two GIN indexes) is built on zero rows — the DDL is effectively instant and needs +# no AddIndexConcurrently. The data backfill runs separately in 0286 (atomic=False, resumable). +# +# >50M-row fallback (documented, NOT the default path): on installs where the subsequent 0286 +# backfill would insert tens of millions of reference rows, GIN maintenance during the backfill +# dominates. Such installs can strip the two GinIndex entries from the VulnerabilityId Meta here, +# let 0286 populate the tables, then add both GIN indexes with CREATE INDEX CONCURRENTLY in a +# follow-up migration. Not needed for typical installs. + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +import django.core.validators +import django.db.models.deletion +import django.db.models.functions.text +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('dojo', '0284_backfill_finding_cwe'), + ] + + operations = [ + migrations.CreateModel( + name='VulnerabilityId', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('vulnerability_id', models.TextField(unique=True, verbose_name='Vulnerability Id')), + ('vulnerability_id_type', models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True)), + ('epss_score', models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), + ('epss_percentile', models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), + ('known_exploited', models.BooleanField(blank=True, default=None, null=True)), + ('ransomware_used', models.BooleanField(blank=True, default=None, null=True)), + ('kev_date', models.DateField(blank=True, default=None, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('updated', models.DateTimeField(auto_now=True)), + ], + options={ + 'indexes': [django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector('vulnerability_id', config='english', weight='A'), name='dojo_vulnid_entity_fts_gin'), django.contrib.postgres.indexes.GinIndex(fields=['vulnerability_id'], name='dojo_vulnid_entity_trgm', opclasses=['gin_trgm_ops']), models.Index(django.db.models.functions.text.Upper('vulnerability_id'), name='dojo_vulnid_entity_upper')], + }, + ), + migrations.CreateModel( + name='FindingVulnerabilityReference', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveSmallIntegerField(default=0, help_text="Position in the finding's id list; 0 is the primary identifier.")), + ('created', models.DateTimeField(auto_now_add=True)), + ('finding', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='vulnerability_references', to='dojo.finding')), + ('vulnerability', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='finding_references', to='dojo.vulnerabilityid')), + ], + options={ + 'ordering': ['order'], + 'indexes': [models.Index(fields=['finding', 'order'], name='dojo_findin_finding_d1a7a8_idx'), models.Index(fields=['vulnerability'], name='dojo_findin_vulnera_13169c_idx')], + 'constraints': [models.UniqueConstraint(fields=('finding', 'vulnerability'), name='unique_finding_vulnerability')], + }, + ), + ] diff --git a/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py b/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py new file mode 100644 index 00000000000..0b1e0c4a067 --- /dev/null +++ b/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py @@ -0,0 +1,42 @@ +# Data-only backfill of the VulnerabilityId entity + FindingVulnerabilityReference tables created +# in 0285. atomic=False so each set-based statement (and each finding_id window in the reference +# build) commits on its own — a crash mid-backfill resumes where it left off, and ON CONFLICT DO +# NOTHING everywhere makes any re-run a no-op on completed work. Insert-only into the two NEW +# tables: dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted, so +# concurrent readers/writers are not blocked. +# +# The actual work lives in dojo.vulnerability_id.backfill.run_backfill, shared with the +# `migrate_vulnerability_ids` management command (big installs pre-run the command on the previous +# release; this migration is then an idempotent catch-up during the deploy). +import logging + +from django.db import connection, migrations + +logger = logging.getLogger(__name__) + + +def backfill_vulnerability_id_entities(apps, schema_editor): + if connection.vendor != "postgresql": + logger.warning( + "Vulnerability ID entity backfill is PostgreSQL-only and was skipped on the %s " + "backend. Run `manage.py migrate_vulnerability_ids` for a backend-appropriate backfill.", + connection.vendor, + ) + return + from dojo.vulnerability_id.backfill import run_backfill # noqa: PLC0415 -- keep entity import lazy/out of migration graph build + run_backfill(logger=logger) + + +class Migration(migrations.Migration): + + """Idempotent, resumable, insert-only backfill of the vulnerability-id entity tables.""" + + atomic = False + + dependencies = [ + ("dojo", "0285_vulnerability_id_entity_tables"), + ] + + operations = [ + migrations.RunPython(backfill_vulnerability_id_entities, migrations.RunPython.noop), + ] diff --git a/dojo/vulnerability_id/backfill.py b/dojo/vulnerability_id/backfill.py new file mode 100644 index 00000000000..e7b1ee1a00a --- /dev/null +++ b/dojo/vulnerability_id/backfill.py @@ -0,0 +1,140 @@ +""" +Idempotent, set-based backfill of the VulnerabilityId entity + FindingVulnerabilityReference +tables from the legacy dojo_vulnerability_id table and dojo_finding.cve. + +Shared by migration 0286 (RunPython forward) and the `migrate_vulnerability_ids` management +command. PostgreSQL only (uses ``ON CONFLICT``); callers guard on vendor and print the escape +hatch for other backends. Every statement is insert-only into the two NEW tables with +``ON CONFLICT DO NOTHING`` — nothing in dojo_vulnerability_id / dojo_finding is ever read-locked, +rewritten, or deleted, and any re-run (migration or command) is a no-op on completed work. + +The reference `order` is assigned cve-first then legacy-PK-ascending, so ``order == 0`` is the +finding's cve (primary id) by construction — matching every live write path. +""" +import logging + +from django.db import connection + +from dojo.finding.vulnerability_id import resolve_vulnerability_id_type + +DEFAULT_WINDOW_SIZE = 100_000 +_TYPE_UPDATE_BATCH = 1000 + +_INSERT_ENTITIES_FROM_LEGACY = """ + INSERT INTO dojo_vulnerabilityid (vulnerability_id, vulnerability_id_type, created, updated) + SELECT vulnerability_id, MAX(vulnerability_id_type), now(), now() + FROM dojo_vulnerability_id + GROUP BY vulnerability_id + ON CONFLICT (vulnerability_id) DO NOTHING +""" + +_INSERT_ENTITIES_FROM_CVE = """ + INSERT INTO dojo_vulnerabilityid (vulnerability_id, vulnerability_id_type, created, updated) + SELECT DISTINCT cve, NULL, now(), now() + FROM dojo_finding + WHERE cve IS NOT NULL AND cve != '' + ON CONFLICT (vulnerability_id) DO NOTHING +""" + +# Windowed by legacy finding_id. ROW_NUMBER orders cve-match first, then legacy PK ascending; +# s.ord - 1 makes order 0-based so the cve row lands at order 0. The join to dojo_vulnerabilityid +# resolves each legacy string to its entity PK. ON CONFLICT covers re-runs and the unique +# (finding_id, vulnerability_id) constraint (dojo 0282 guarantees legacy pair uniqueness, so no +# dedupe layer is needed here). +_INSERT_REFERENCES_WINDOW = """ + INSERT INTO dojo_findingvulnerabilityreference (finding_id, vulnerability_id, "order", created) + SELECT s.finding_id, e.id, s.ord - 1, now() + FROM ( + SELECT v.finding_id, v.vulnerability_id, + ROW_NUMBER() OVER ( + PARTITION BY v.finding_id + ORDER BY (v.vulnerability_id = f.cve) DESC, v.id ASC + ) AS ord + FROM dojo_vulnerability_id v + JOIN dojo_finding f ON f.id = v.finding_id + WHERE v.finding_id >= %(lo)s AND v.finding_id < %(hi)s + ) s + JOIN dojo_vulnerabilityid e ON e.vulnerability_id = s.vulnerability_id + ON CONFLICT (finding_id, vulnerability_id) DO NOTHING +""" + +# Findings that carry a cve but have NO legacy vulnerability_id rows (so the windowed pass above +# created no reference for them): give each an order-0 reference to its cve entity. +_INSERT_CVE_ONLY_REFERENCES = """ + INSERT INTO dojo_findingvulnerabilityreference (finding_id, vulnerability_id, "order", created) + SELECT f.id, e.id, 0, now() + FROM dojo_finding f + JOIN dojo_vulnerabilityid e ON e.vulnerability_id = f.cve + LEFT JOIN dojo_findingvulnerabilityreference r ON r.finding_id = f.id + WHERE f.cve IS NOT NULL AND f.cve != '' AND r.id IS NULL + ON CONFLICT (finding_id, vulnerability_id) DO NOTHING +""" + + +def _stamp_missing_types(cursor, logger): + """ + Populate vulnerability_id_type for entities where it is still NULL. + + resolve_vulnerability_id_type is a pure Python prefix parse (not expressible in SQL), so we + fetch untyped rows and UPDATE in batches. Rows whose id resolves to None (bare numbers / no + prefix) are left NULL — that is their permanent, correct value, and skipping them keeps the + pass idempotent (a re-run produces zero UPDATEs). + """ + cursor.execute( + "SELECT id, vulnerability_id FROM dojo_vulnerabilityid WHERE vulnerability_id_type IS NULL", + ) + rows = cursor.fetchall() + updates = [ + (resolved, pk) + for pk, vuln_id in rows + if (resolved := resolve_vulnerability_id_type(vuln_id)) is not None + ] + stamped = 0 + for i in range(0, len(updates), _TYPE_UPDATE_BATCH): + batch = updates[i:i + _TYPE_UPDATE_BATCH] + cursor.executemany( + "UPDATE dojo_vulnerabilityid SET vulnerability_id_type = %s WHERE id = %s", + batch, + ) + stamped += len(batch) + logger.info(" stamped vulnerability_id_type on %s entities", stamped) + return stamped + + +def run_backfill(*, logger=None, window_size=DEFAULT_WINDOW_SIZE): + """ + Run the full idempotent backfill. PostgreSQL only. Returns a dict of per-step row counts. + + Callers (migration 0286 / migrate_vulnerability_ids) are responsible for the vendor guard. + """ + logger = logger or logging.getLogger(__name__) + counts = {} + + with connection.cursor() as cursor: + logger.info("Backfill step 1/4: entities from legacy dojo_vulnerability_id") + cursor.execute(_INSERT_ENTITIES_FROM_LEGACY) + counts["entities_from_legacy"] = cursor.rowcount + + logger.info("Backfill step 2/4: cve-only entities from dojo_finding.cve") + cursor.execute(_INSERT_ENTITIES_FROM_CVE) + counts["entities_from_cve"] = cursor.rowcount + counts["types_stamped"] = _stamp_missing_types(cursor, logger) + + logger.info("Backfill step 3/4: references (windowed by finding_id, window=%s)", window_size) + cursor.execute("SELECT MIN(finding_id), MAX(finding_id) FROM dojo_vulnerability_id") + lo, hi_max = cursor.fetchone() + references = 0 + if lo is not None: + while lo <= hi_max: + hi = lo + window_size + cursor.execute(_INSERT_REFERENCES_WINDOW, {"lo": lo, "hi": hi}) + references += cursor.rowcount + lo = hi + counts["references_from_legacy"] = references + + logger.info("Backfill step 4/4: order-0 references for cve-only findings") + cursor.execute(_INSERT_CVE_ONLY_REFERENCES) + counts["references_from_cve_only"] = cursor.rowcount + + logger.info("Backfill complete: %s", counts) + return counts diff --git a/unittests/test_migrations.py b/unittests/test_migrations.py index 28e7a013f4d..42e2cf9b98e 100644 --- a/unittests/test_migrations.py +++ b/unittests/test_migrations.py @@ -2,9 +2,24 @@ from unittest import skip from django.contrib.auth import get_user_model +from django.test import TestCase from django.utils import timezone from django_test_migrations.contrib.unittest_case import MigratorTestCase +from dojo.finding.vulnerability_id import resolve_vulnerability_id_type +from dojo.models import ( + Engagement, + Finding, + FindingVulnerabilityReference, + Product, + Product_Type, + Test, + Test_Type, + Vulnerability_Id, + VulnerabilityId, +) +from dojo.vulnerability_id.backfill import run_backfill + @skip("Outdated - this class was testing some version of migration; it is not needed anymore") class TestOptiEndpointStatus(MigratorTestCase): @@ -161,3 +176,146 @@ def test_after_migration(self): ) self.assertEqual(eps.all().count(), 1, ep) self.assertIsInstance(eps.all().first(), Endpoint_Status) + + +class TestVulnerabilityIdBackfill(TestCase): + + """ + dojo.vulnerability_id.backfill.run_backfill (the routine migration 0286 delegates to) turns + legacy dojo_vulnerability_id rows + dojo_finding.cve into VulnerabilityId entities and ordered + FindingVulnerabilityReference rows (order 0 == cve). Covers ordered ids, cve-drift (cve is not + the lowest-PK row), case-variant ids (2 distinct entities), cve-only findings (no legacy rows), + null-cve findings, and an idempotent re-run. + + A plain TestCase rather than MigratorTestCase: django_test_migrations replays the full squashed + migration chain from a wiped schema in setUp, which is unusable in this codebase (the replay + collides on tables such as dojo_cred_user whose model no longer matches the squash) — the very + reason the only other MigratorTestCase here is skipped. run_backfill IS what 0286 runs, and the + migration's application is separately covered by the create-db and dev-DB migrate runs. + """ + + @classmethod + def setUpTestData(cls): + now = timezone.now() + prod_type = Product_Type.objects.create(name="vulnid-pt") + product = Product.objects.create(prod_type=prod_type, name="vulnid-prod") + engagement = Engagement.objects.create(product=product, target_start=now, target_end=now) + test_type = Test_Type.objects.create(name="vulnid-tt") + test = Test.objects.create( + engagement=engagement, + test_type=test_type, + target_start=now, + target_end=now, + ) + user = get_user_model().objects.create(username="vulnid-user") + + def make_finding(title, cve): + return Finding.objects.create( + test=test, + reporter=user, + title=title, + description=title, + severity="High", + numerical_severity="S0", + active=True, + verified=False, + cve=cve, + ) + + def add_legacy(finding, vulnerability_id): + # Real save() stamps vulnerability_id_type, mirroring production legacy rows. + Vulnerability_Id.objects.create(finding=finding, vulnerability_id=vulnerability_id) + + # A: ordered ids, cve == first (lowest-PK) row. + cls.finding_a = make_finding("A", "CVE-A1") + for vid in ("CVE-A1", "CVE-A2", "CVE-A3"): + add_legacy(cls.finding_a, vid) + + # B: cve-drift — rows created B1,B2,B3 (ascending PK) but cve == CVE-B2 (middle row). + cls.finding_b = make_finding("B", "CVE-B2") + for vid in ("CVE-B1", "CVE-B2", "CVE-B3"): + add_legacy(cls.finding_b, vid) + + # C: case-variant ids — CVE-C1 and cve-c1 are distinct exact-cased strings => 2 entities. + cls.finding_c = make_finding("C", "CVE-C1") + for vid in ("CVE-C1", "cve-c1"): + add_legacy(cls.finding_c, vid) + + # D: cve-only — cve set, NO legacy rows => step-4 order-0 ref to the cve entity. + cls.finding_d = make_finding("D", "CVE-D1") + + # E: legacy rows but NULL cve => order by PK, no cve-only ref. + cls.finding_e = make_finding("E", None) + for vid in ("CVE-E1", "CVE-E2"): + add_legacy(cls.finding_e, vid) + + def _order_map(self, finding): + return { + ref.order: ref.vulnerability.vulnerability_id + for ref in FindingVulnerabilityReference.objects.filter(finding=finding) + } + + def test_backfill(self): + run_backfill() + + expected_entities = { + "CVE-A1", "CVE-A2", "CVE-A3", + "CVE-B1", "CVE-B2", "CVE-B3", + "CVE-C1", "cve-c1", + "CVE-D1", + "CVE-E1", "CVE-E2", + } + + with self.subTest("entity extraction (exact-cased, case-variant = 2 entities)"): + self.assertEqual( + set(VulnerabilityId.objects.values_list("vulnerability_id", flat=True)), + expected_entities, + ) + + with self.subTest("entity type carried / stamped from prefix"): + for entity in VulnerabilityId.objects.all(): + self.assertEqual( + entity.vulnerability_id_type, + resolve_vulnerability_id_type(entity.vulnerability_id), + ) + + with self.subTest("A: ordered ids, cve first"): + self.assertEqual(self._order_map(self.finding_a), {0: "CVE-A1", 1: "CVE-A2", 2: "CVE-A3"}) + + with self.subTest("B: cve-drift resolves cve to order 0, rest by PK asc"): + self.assertEqual(self._order_map(self.finding_b), {0: "CVE-B2", 1: "CVE-B1", 2: "CVE-B3"}) + + with self.subTest("C: case-variant ids both referenced, exact cve at order 0"): + self.assertEqual(self._order_map(self.finding_c), {0: "CVE-C1", 1: "cve-c1"}) + + with self.subTest("D: cve-only finding gets an order-0 reference"): + self.assertEqual(self._order_map(self.finding_d), {0: "CVE-D1"}) + + with self.subTest("E: null-cve finding ordered by PK, no cve-only ref"): + self.assertEqual(self._order_map(self.finding_e), {0: "CVE-E1", 1: "CVE-E2"}) + + with self.subTest("pair sets: every legacy pair is in the new store; extras are cve-only"): + legacy_pairs = set(Vulnerability_Id.objects.values_list("finding_id", "vulnerability_id")) + new_pairs = { + (ref.finding_id, ref.vulnerability.vulnerability_id) + for ref in FindingVulnerabilityReference.objects.all() + } + self.assertTrue(legacy_pairs.issubset(new_pairs)) + self.assertEqual(new_pairs - legacy_pairs, {(self.finding_d.id, "CVE-D1")}) + + with self.subTest("idempotency: re-running the backfill changes nothing"): + entities_before = VulnerabilityId.objects.count() + refs_before = FindingVulnerabilityReference.objects.count() + counts = run_backfill() + self.assertEqual(VulnerabilityId.objects.count(), entities_before) + self.assertEqual(FindingVulnerabilityReference.objects.count(), refs_before) + self.assertEqual( + counts, + { + "entities_from_legacy": 0, + "entities_from_cve": 0, + "types_stamped": 0, + "references_from_legacy": 0, + "references_from_cve_only": 0, + }, + ) From bae2a4397916393f7cf374c16872ff8feb824ee3 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:32:07 -0600 Subject: [PATCH 03/18] feat(vuln-id): unconditional dual-write through VulnerabilityIdManager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every vulnerability-id writer now persists the legacy Vulnerability_Id rows byte-for-byte AND the new VulnerabilityId entity + FindingVulnerabilityReference rows, through a single seam. The flag gates reads only (WP4), so the two stores never drift. Legacy-write blocks are marked "TODO: Delete after Vulnerability_Id retirement". - dojo/vulnerability_id/manager.py: bulk_get_or_create_entities (race-safe bulk_create(ignore_conflicts)+refetch), persist_for_finding (single-finding core; does NOT touch finding.cve — caller keeps the cve sync), and VulnerabilityIdManager (importer accumulate/flush; record / record_reconcile / flush in one transaction). order == enumerate position => order 0 is the cve. - base_importer: store_vulnerability_ids -> manager.record; flush_vulnerability_ids -> manager.flush() then the unchanged Finding_CWE flush (CWE buffers are NOT owned by the manager and keep riding this flush boundary). - default_reimporter.reconcile_vulnerability_ids: keep the prefetched set-equality early-exit (legacy store authoritative) + cve sync; on change -> record_reconcile. - finding/helper.save_vulnerability_ids: dedupe/sanitize + cve sync unchanged; storage delegates to persist_for_finding. (migrate_cve stays legacy-only.) Tests: test_importers_importer updated to assert the manager's buffers (46 green); new fixture-free test_vulnerability_id asserts the entity-side dual-write, order, reconcile, orphan retention, and empty-clears for persist_for_finding, the manager, and save_vulnerability_ids; test_finding_helper.test_save_vulnerability_ids now asserts delegation to persist_for_finding + cve sync. Verified locally: test_importers_importer + test_vulnerability_id + TestSaveVulnerabilityIds green; manual import/reimport smoke on the dev DB shows both stores mirrored with correct order and orphan entities retained. The fixture-based suites (import_reimport / rest_framework / deduplication_logic / finding_helper API) run in OSS CI (dojo_testdata.json can't load in the Pro container: watson.searchentry). Co-Authored-By: Claude Fable 5 --- dojo/finding/helper.py | 15 +-- dojo/importers/base_importer.py | 24 ++--- dojo/importers/default_reimporter.py | 10 +- dojo/vulnerability_id/manager.py | 144 +++++++++++++++++++++++++++ unittests/test_finding_helper.py | 18 ++-- unittests/test_importers_importer.py | 19 ++-- unittests/test_vulnerability_id.py | 126 +++++++++++++++++++++++ 7 files changed, 301 insertions(+), 55 deletions(-) create mode 100644 dojo/vulnerability_id/manager.py create mode 100644 unittests/test_vulnerability_id.py diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index 6724135337f..85586ba252a 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -27,7 +27,6 @@ do_false_positive_history_batch, get_finding_models_for_deduplication, ) -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.jira import services as jira_services from dojo.location.models import Location from dojo.location.status import FindingLocationStatus @@ -44,7 +43,6 @@ Notes, System_Settings, Test, - Vulnerability_Id, ) from dojo.notes.helper import delete_related_notes from dojo.notifications.helper import create_notification @@ -57,6 +55,7 @@ get_object_or_none, to_str_typed, ) +from dojo.vulnerability_id.manager import persist_for_finding logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -1064,16 +1063,10 @@ def save_vulnerability_ids(finding, vulnerability_ids, *, delete_existing: bool vulnerability_ids = list(dict.fromkeys(vulnerability_ids)) vulnerability_ids = sanitize_vulnerability_ids(vulnerability_ids) - # Remove old vulnerability ids if requested + # Unconditional dual-write of both stores (legacy Vulnerability_Id rows + entity references). # Callers can set delete_existing=False when they know there are no existing IDs - # to avoid an unnecessary delete query (e.g., for new findings) - if delete_existing: - Vulnerability_Id.objects.filter(finding=finding).delete() - - Vulnerability_Id.objects.bulk_create([ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) - for vid in vulnerability_ids - ]) + # to avoid an unnecessary delete query (e.g., for new findings). + persist_for_finding(finding, vulnerability_ids, delete_existing=delete_existing) # Set CVE if vulnerability_ids: diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index 8573e05573c..d2afbe8cd8c 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -13,7 +13,6 @@ import dojo.finding.helper as finding_helper import dojo.risk_acceptance.helper as ra_helper from dojo.finding.cwe import finding_cwe_labels -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.importers.options import ImporterOptions from dojo.jira.services import is_keep_in_sync from dojo.location.models import Location @@ -36,13 +35,13 @@ Test_Import, Test_Import_Finding_Action, Test_Type, - Vulnerability_Id, ) from dojo.notifications.helper import create_notification from dojo.tags.utils import bulk_add_tags_to_instances from dojo.tools.factory import get_parser from dojo.tools.parser_test import ParserTest from dojo.utils import max_safe +from dojo.vulnerability_id.manager import VulnerabilityIdManager logger = logging.getLogger(__name__) @@ -83,8 +82,9 @@ def __init__( and will raise a `NotImplemented` exception """ ImporterOptions.__init__(self, *args, **kwargs) - self.pending_vulnerability_ids: list[Vulnerability_Id] = [] - self.pending_vuln_id_deletes: list[int] = [] + # Unconditional dual-write seam: buffers legacy Vulnerability_Id rows AND the new + # entity/reference rows, flushed together at the batch boundary. + self.vulnerability_id_manager = VulnerabilityIdManager() self.pending_cwes: list[Finding_CWE] = [] self.pending_cwe_deletes: list[int] = [] self.pending_burp_rr: list[BurpRawRequestResponse] = [] @@ -897,10 +897,7 @@ def store_vulnerability_ids( self.sanitize_vulnerability_ids(finding) vulnerability_ids_to_process = list(dict.fromkeys(finding.unsaved_vulnerability_ids or [])) vulnerability_ids_to_process = [x for x in vulnerability_ids_to_process if x.strip()] - self.pending_vulnerability_ids.extend([ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) - for vid in vulnerability_ids_to_process - ]) + self.vulnerability_id_manager.record(finding, vulnerability_ids_to_process) if vulnerability_ids_to_process: finding.cve = vulnerability_ids_to_process[0] else: @@ -929,13 +926,10 @@ def reconcile_cwes(self, finding: Finding) -> None: self.pending_cwes.extend([Finding_CWE(finding=finding, cwe=cwe) for cwe in new_cwes]) def flush_vulnerability_ids(self) -> None: - """Delete stale and bulk-insert accumulated Vulnerability_Id / Finding_CWE objects, then clear buffers.""" - if self.pending_vuln_id_deletes: - Vulnerability_Id.objects.filter(finding_id__in=self.pending_vuln_id_deletes).delete() - self.pending_vuln_id_deletes.clear() - if self.pending_vulnerability_ids: - Vulnerability_Id.objects.bulk_create(self.pending_vulnerability_ids, batch_size=1000) - self.pending_vulnerability_ids.clear() + """Flush the dual-write vulnerability-id buffers, then the Finding_CWE buffers, and clear.""" + # Legacy Vulnerability_Id rows + entity/reference rows, in one transaction. + self.vulnerability_id_manager.flush() + # CWE buffers ride the same flush boundary as before (not owned by the manager). if self.pending_cwe_deletes: Finding_CWE.objects.filter(finding_id__in=self.pending_cwe_deletes).delete() self.pending_cwe_deletes.clear() diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index 7284f422bcb..f6ea16006ba 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -13,7 +13,6 @@ find_candidates_for_deduplication_unique_id, find_candidates_for_reimport_legacy, ) -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.importers.base_importer import BaseImporter, Parser from dojo.importers.base_location_manager import LocationHandler from dojo.importers.options import ImporterOptions @@ -25,7 +24,6 @@ Notes, Test, Test_Import, - Vulnerability_Id, ) from dojo.tags import inheritance as tag_inheritance from dojo.tags.inheritance import apply_inherited_tags_for_findings @@ -1002,12 +1000,8 @@ def reconcile_vulnerability_ids( ) return finding - # Accumulate delete + insert for batch flush - self.pending_vuln_id_deletes.append(finding.id) - self.pending_vulnerability_ids.extend([ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) - for vid in vulnerability_ids_to_process - ]) + # Accumulate delete + insert for batch flush (legacy rows + entity refs, dual-write). + self.vulnerability_id_manager.record_reconcile(finding, vulnerability_ids_to_process) if vulnerability_ids_to_process: finding.cve = vulnerability_ids_to_process[0] else: diff --git a/dojo/vulnerability_id/manager.py b/dojo/vulnerability_id/manager.py new file mode 100644 index 00000000000..c7e7c1256bd --- /dev/null +++ b/dojo/vulnerability_id/manager.py @@ -0,0 +1,144 @@ +""" +Write seam for vulnerability ids: an UNCONDITIONAL dual-write. + +Every writer persists the legacy ``Vulnerability_Id`` rows byte-for-byte as today AND the new +``VulnerabilityId`` entity + ``FindingVulnerabilityReference`` rows, through this one module. The +``V3_FEATURE_VULNERABILITY_IDS`` flag gates READS only (see dojo.vulnerability_id.queries), so the +two stores never drift and the flag is reversible in both directions. + +The legacy-write blocks are marked ``# TODO: Delete after Vulnerability_Id retirement``. +CWE buffering (store_cwes/pending_cwes/reconcile_cwes) is deliberately NOT owned here — it keeps +riding the importer flush boundary exactly as before. +""" +from collections.abc import Iterable + +from django.db import transaction + +from dojo.finding.models import Vulnerability_Id +from dojo.finding.vulnerability_id import resolve_vulnerability_id_type +from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId + + +def _clean(vulnerability_ids: Iterable[str]) -> list[str]: + """Dedupe (order-preserving) and drop empty/whitespace ids — the list order IS the ref order.""" + return [vid for vid in dict.fromkeys(vulnerability_ids or []) if vid and vid.strip()] + + +def bulk_get_or_create_entities(strings: Iterable[str]) -> dict[str, int]: + """ + Map each id string to its ``VulnerabilityId`` PK, creating any that are missing. + + Race-safe by design (a deliberate departure from the locations rollback-on-race pattern): + shared global id strings collide routinely across concurrent imports, so we + ``bulk_create(ignore_conflicts=True)`` the missing rows and then refetch — whoever wins the + insert, every caller converges on the one surviving PK. There is no orphanable parent here. + """ + unique = _clean(strings) + if not unique: + return {} + mapping = dict( + VulnerabilityId.objects.filter(vulnerability_id__in=unique).values_list("vulnerability_id", "id"), + ) + missing = [s for s in unique if s not in mapping] + if missing: + VulnerabilityId.objects.bulk_create( + [ + VulnerabilityId(vulnerability_id=s, vulnerability_id_type=resolve_vulnerability_id_type(s)) + for s in missing + ], + ignore_conflicts=True, + batch_size=1000, + ) + # bulk_create(ignore_conflicts=True) does not return usable PKs on Postgres — refetch. + mapping.update( + dict( + VulnerabilityId.objects.filter(vulnerability_id__in=missing).values_list("vulnerability_id", "id"), + ), + ) + return mapping + + +def _write_references(pending): + """Replace references for the given (finding, ids) pairs from a freshly built entity map.""" + entity_map = bulk_get_or_create_entities([vid for _, ids in pending for vid in ids]) + references = [ + FindingVulnerabilityReference(finding=finding, vulnerability_id=entity_map[vid], order=order) + for finding, ids in pending + for order, vid in enumerate(ids) + ] + if references: + # ignore_conflicts guards only against a concurrent racer having inserted the same + # (finding, entity) pair; the caller has already deleted this finding's existing refs. + FindingVulnerabilityReference.objects.bulk_create(references, batch_size=1000, ignore_conflicts=True) + + +def persist_for_finding(finding, vulnerability_ids, *, delete_existing: bool = True) -> None: + """ + Single-finding dual-write core (no buffering). Does NOT touch ``finding.cve`` — the cve sync + stays with the caller (e.g. save_vulnerability_ids) exactly as before. + """ + ids = _clean(vulnerability_ids) + with transaction.atomic(): + # --- LEGACY store (byte-for-byte today's behavior) --- + # TODO: Delete after Vulnerability_Id retirement + if delete_existing: + Vulnerability_Id.objects.filter(finding=finding).delete() + Vulnerability_Id.objects.bulk_create( + [ + Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) + for vid in ids + ], + batch_size=1000, + ) + # --- Entity + reference store --- + if delete_existing: + FindingVulnerabilityReference.objects.filter(finding=finding).delete() + _write_references([(finding, ids)]) + + +class VulnerabilityIdManager: + + """ + Accumulate/flush dual-writes for a batch of findings (importer path). + + Replaces the importer's ``pending_vulnerability_ids`` / ``pending_vuln_id_deletes`` buffers. + ``record`` is for new findings (pure insert); ``record_reconcile`` is for reimported findings + whose ids changed (delete-then-insert). ``flush`` commits the whole batch in one transaction. + """ + + def __init__(self): + self.pending: list[tuple[object, list[str]]] = [] + self.pending_deletes: list[int] = [] + + def record(self, finding, vulnerability_ids) -> None: + """Buffer a new finding's ids (no existing rows to delete).""" + self.pending.append((finding, _clean(vulnerability_ids))) + + def record_reconcile(self, finding, vulnerability_ids) -> None: + """Buffer a changed finding: delete its existing rows/refs, then insert the new ids.""" + self.pending_deletes.append(finding.id) + self.pending.append((finding, _clean(vulnerability_ids))) + + def flush(self) -> None: + """Delete stale and bulk-insert the buffered legacy rows + entity refs, then clear.""" + if not self.pending and not self.pending_deletes: + return + with transaction.atomic(): + # --- LEGACY store (byte-for-byte today's behavior) --- + # TODO: Delete after Vulnerability_Id retirement + if self.pending_deletes: + Vulnerability_Id.objects.filter(finding_id__in=self.pending_deletes).delete() + legacy_rows = [ + Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) + for finding, ids in self.pending + for vid in ids + ] + if legacy_rows: + Vulnerability_Id.objects.bulk_create(legacy_rows, batch_size=1000) + # --- Entity + reference store --- + # Delete refs for reimported (changed) findings; new findings have none. + if self.pending_deletes: + FindingVulnerabilityReference.objects.filter(finding_id__in=self.pending_deletes).delete() + _write_references(self.pending) + self.pending.clear() + self.pending_deletes.clear() diff --git a/unittests/test_finding_helper.py b/unittests/test_finding_helper.py index f6d9e747ff2..811ebf0c809 100644 --- a/unittests/test_finding_helper.py +++ b/unittests/test_finding_helper.py @@ -10,7 +10,7 @@ from rest_framework.test import APIClient from dojo.finding.helper import save_vulnerability_ids, save_vulnerability_ids_template -from dojo.models import Finding, Finding_Template, Test, Vulnerability_Id +from dojo.models import Finding, Finding_Template, Test from unittests.dojo_test_case import DojoAPITestCase, DojoTestCase, versioned_fixtures logger = logging.getLogger(__name__) @@ -218,22 +218,16 @@ def test_set_active_as_out_of_scope(self, mock_can_edit, mock_tz): class TestSaveVulnerabilityIds(DojoTestCase): - @patch("dojo.finding.helper.Vulnerability_Id.objects.filter") - @patch("django.db.models.query.QuerySet.delete") - @patch("dojo.finding.helper.Vulnerability_Id.objects.bulk_create") - def test_save_vulnerability_ids(self, bulk_create_mock, delete_mock, filter_mock): + @patch("dojo.finding.helper.persist_for_finding") + def test_save_vulnerability_ids(self, persist_mock): finding = Finding() new_vulnerability_ids = ["REF-1", "REF-2", "REF-2"] - filter_mock.return_value = Vulnerability_Id.objects.none() save_vulnerability_ids(finding, new_vulnerability_ids) - filter_mock.assert_called_with(finding=finding) - delete_mock.assert_called_once() - bulk_create_mock.assert_called_once() - # Duplicates are removed: REF-1 and REF-2 only - created_objects = bulk_create_mock.call_args[0][0] - self.assertEqual(2, len(created_objects)) + # Delegates the (dual) write to persist_for_finding with deduped/sanitized ids... + persist_mock.assert_called_once_with(finding, ["REF-1", "REF-2"], delete_existing=True) + # ...and keeps the cve sync (first id) in the helper. self.assertEqual("REF-1", finding.cve) @patch("dojo.models.Finding_Template.save") diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py index 6b82624cab0..0b38f132c34 100644 --- a/unittests/test_importers_importer.py +++ b/unittests/test_importers_importer.py @@ -1171,11 +1171,12 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): reimporter.reconcile_vulnerability_ids(finding_b) reimporter.reconcile_vulnerability_ids(finding_c) - # pending_vuln_id_deletes only contains changed findings, not finding_c - self.assertIn(finding_a.id, reimporter.pending_vuln_id_deletes) - self.assertIn(finding_b.id, reimporter.pending_vuln_id_deletes) - self.assertNotIn(finding_c.id, reimporter.pending_vuln_id_deletes) - self.assertEqual(2, len(reimporter.pending_vulnerability_ids)) + # pending_deletes only contains changed findings, not finding_c + manager = reimporter.vulnerability_id_manager + self.assertIn(finding_a.id, manager.pending_deletes) + self.assertIn(finding_b.id, manager.pending_deletes) + self.assertNotIn(finding_c.id, manager.pending_deletes) + self.assertEqual(2, sum(len(ids) for _, ids in manager.pending)) # Old IDs still in DB (not yet deleted) self.assertEqual(1, Vulnerability_Id.objects.filter(finding=finding_a).count()) @@ -1184,8 +1185,8 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): reimporter.flush_vulnerability_ids() # Buffers cleared - self.assertEqual([], reimporter.pending_vuln_id_deletes) - self.assertEqual([], reimporter.pending_vulnerability_ids) + self.assertEqual([], reimporter.vulnerability_id_manager.pending_deletes) + self.assertEqual([], reimporter.vulnerability_id_manager.pending) # finding_a: old deleted, new inserted vuln_ids_a = list(Vulnerability_Id.objects.filter(finding=finding_a).values_list("vulnerability_id", flat=True)) @@ -1218,8 +1219,8 @@ def test_reconcile_vulnerability_ids_unchanged_no_db_write(self): finding.unsaved_vulnerability_ids = ["CVE-2020-1234"] reimporter.reconcile_vulnerability_ids(finding) - self.assertEqual([], reimporter.pending_vuln_id_deletes) - self.assertEqual([], reimporter.pending_vulnerability_ids) + self.assertEqual([], reimporter.vulnerability_id_manager.pending_deletes) + self.assertEqual([], reimporter.vulnerability_id_manager.pending) finding.delete() diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py new file mode 100644 index 00000000000..5433fc64635 --- /dev/null +++ b/unittests/test_vulnerability_id.py @@ -0,0 +1,126 @@ +""" +Unconditional dual-write regression tests for the vulnerability-id entity store. + +Fixture-free (no dojo_testdata.json) so they run in both the OSS CI DB and the Pro container. +They assert the NEW behavior that the legacy Vulnerability_Id tests do not: that persist_for_finding +and VulnerabilityIdManager write the VulnerabilityId entity + ordered FindingVulnerabilityReference +rows alongside the legacy rows, keep order 0 == first id, reconcile both stores, and never delete +entity rows (orphans are retained). +""" +from django.contrib.auth import get_user_model +from django.test import TestCase +from django.utils.timezone import now + +from dojo.finding.helper import save_vulnerability_ids +from dojo.models import ( + Engagement, + Finding, + FindingVulnerabilityReference, + Product, + Product_Type, + Test, + Test_Type, + Vulnerability_Id, + VulnerabilityId, +) +from dojo.vulnerability_id.manager import VulnerabilityIdManager, persist_for_finding + + +class VulnerabilityIdDualWriteTestCase(TestCase): + + """Shared minimal Product -> Engagement -> Test scaffold for dual-write tests.""" + + @classmethod + def setUpTestData(cls): + cls.prod_type = Product_Type.objects.create(name="vulnid-dw-pt") + cls.product = Product.objects.create(prod_type=cls.prod_type, name="vulnid-dw-prod") + cls.engagement = Engagement.objects.create(product=cls.product, target_start=now(), target_end=now()) + cls.test_type = Test_Type.objects.create(name="vulnid-dw-tt") + cls.test = Test.objects.create( + engagement=cls.engagement, test_type=cls.test_type, target_start=now(), target_end=now(), + ) + cls.user = get_user_model().objects.create(username="vulnid-dw-user") + + def _make_finding(self, title): + return Finding.objects.create( + test=self.test, reporter=self.user, title=title, severity="High", numerical_severity="S0", + ) + + def _legacy(self, finding): + return sorted(Vulnerability_Id.objects.filter(finding=finding).values_list("vulnerability_id", flat=True)) + + def _refs(self, finding): + return { + ref.order: ref.vulnerability.vulnerability_id + for ref in FindingVulnerabilityReference.objects.filter(finding=finding) + } + + +class TestPersistForFinding(VulnerabilityIdDualWriteTestCase): + + def test_dual_write_and_reconcile(self): + finding = self._make_finding("persist") + + persist_for_finding(finding, ["CVE-2021-1", "CVE-2021-2", "CVE-2021-2"], delete_existing=False) + + with self.subTest("legacy rows written (deduped)"): + self.assertEqual(self._legacy(finding), ["CVE-2021-1", "CVE-2021-2"]) + with self.subTest("entity rows created"): + self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-2021-1").exists()) + self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-2021-2").exists()) + with self.subTest("references ordered, order 0 == first id"): + self.assertEqual(self._refs(finding), {0: "CVE-2021-1", 1: "CVE-2021-2"}) + + # Reconcile with changed ids. + persist_for_finding(finding, ["CVE-2021-2", "CVE-2021-9"], delete_existing=True) + + with self.subTest("legacy reconciled"): + self.assertEqual(self._legacy(finding), ["CVE-2021-2", "CVE-2021-9"]) + with self.subTest("references reconciled with new order"): + self.assertEqual(self._refs(finding), {0: "CVE-2021-2", 1: "CVE-2021-9"}) + with self.subTest("dropped id's entity is retained (orphan registry)"): + self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-2021-1").exists()) + + def test_empty_clears_both_stores(self): + finding = self._make_finding("persist-empty") + persist_for_finding(finding, ["CVE-2022-1"], delete_existing=False) + self.assertEqual(self._refs(finding), {0: "CVE-2022-1"}) + + persist_for_finding(finding, [], delete_existing=True) + self.assertEqual(self._legacy(finding), []) + self.assertEqual(self._refs(finding), {}) + + +class TestSaveVulnerabilityIdsIntegration(VulnerabilityIdDualWriteTestCase): + + def test_save_vulnerability_ids_dual_writes_and_syncs_cve(self): + finding = self._make_finding("helper") + save_vulnerability_ids(finding, ["CVE-2023-1", "CVE-2023-2"], delete_existing=False) + + self.assertEqual(self._legacy(finding), ["CVE-2023-1", "CVE-2023-2"]) + self.assertEqual(self._refs(finding), {0: "CVE-2023-1", 1: "CVE-2023-2"}) + self.assertEqual(finding.cve, "CVE-2023-1") + + +class TestVulnerabilityIdManager(VulnerabilityIdDualWriteTestCase): + + def test_batch_record_and_reconcile_flush(self): + finding_new = self._make_finding("mgr-new") + finding_reimport = self._make_finding("mgr-reimport") + # Pre-existing state for the reimported finding. + persist_for_finding(finding_reimport, ["CVE-OLD-1", "CVE-OLD-2"], delete_existing=False) + + manager = VulnerabilityIdManager() + manager.record(finding_new, ["CVE-NEW-1", "CVE-NEW-2"]) + manager.record_reconcile(finding_reimport, ["CVE-OLD-2", "CVE-NEW-3"]) + manager.flush() + + with self.subTest("new finding dual-written"): + self.assertEqual(self._legacy(finding_new), ["CVE-NEW-1", "CVE-NEW-2"]) + self.assertEqual(self._refs(finding_new), {0: "CVE-NEW-1", 1: "CVE-NEW-2"}) + with self.subTest("reimported finding reconciled in both stores"): + self.assertEqual(self._legacy(finding_reimport), ["CVE-NEW-3", "CVE-OLD-2"]) + self.assertEqual(self._refs(finding_reimport), {0: "CVE-OLD-2", 1: "CVE-NEW-3"}) + with self.subTest("buffers cleared after flush"): + self.assertEqual(manager.pending, []) + self.assertEqual(manager.pending_deletes, []) From 1322604101d67ad9cc6cbabc830f41423436f7f4 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 11:55:40 -0600 Subject: [PATCH 04/18] feat(vuln-id): flag-gated entity read seam, wire-compatible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reads switch to the VulnerabilityId entity/reference store when V3_FEATURE_VULNERABILITY_IDS is on (default True); off keeps today's legacy-table reads. The flag is consulted ONLY in dojo/vulnerability_id/queries.py and the two Finding methods. Writes stay unconditionally dual (WP3), so both stores hold identical data and the flip is reversible with no drift. queries.py seam: use_entity_reads, finding_ids_with_vulnerability_ids (in/exact), vulnerability_id_prefetch (with nested prefix), first_vulnerability_id_subquery (order==0), finding_vulnerability_id_strings. Re-pointed (identical output flag-off; entity read flag-on): - Finding.vulnerability_ids property + get_vulnerability_ids() (cve prepend / sorted-join kept byte-for-byte). - dojo/filters.py both filter fns; risk_acceptance exact-match mixin. - The 7 prefetch sites (deduplication, commands/dedupe, finding/ui + test/ui views, finding/queries x2) -> vulnerability_id_prefetch(). - v2 serializer: VulnerabilityIdsField reads [{"vulnerability_id": s}] from the flag-appropriate store (schema pinned via extend_schema_field); create/update pop `parsed_vulnerability_ids` and still funnel writes to save_vulnerability_ids. - Finding.copy() now dual-writes copied ids through persist_for_finding (was a legacy-only .objects.create — would have been invisible flag-on). Deliberately NOT re-pointed this PR (kept on the dual-written legacy table, which stays correct in both flag states; entity migration deferred to the legacy-drop phase): watson registration + the classic OSS search view. Rationale: watson is disabled in the Pro runtime (untestable locally), the classic search view/template are coupled to watson indexing Vulnerability_Id, and the classic UI is being retired. This also removes the buildwatson-on-flip requirement. The Pro Vue global search IS re-pointed to the entity in the Pro PR. Tests (fixture-free, run locally + CI): test_vulnerability_id.TestReadSeamFlagParity asserts the v2 wire shape, hash input (get_vulnerability_ids), the property, and the filter seam are byte-identical flag-off vs flag-on; test_importers_importer reconcile seeds via the dual-write seam so entity reads resolve. Verified green in BOTH flag states locally; the fixture-based suites (rest_framework schema, import_reimport, deduplication_logic, finding_helper API, watson trio) run in OSS CI. Co-Authored-By: Claude Fable 5 --- dojo/apps.py | 4 ++ dojo/filters.py | 12 +--- dojo/finding/api/serializer.py | 76 ++++++++++++++++++------- dojo/finding/deduplication.py | 5 +- dojo/finding/models.py | 35 ++++++++---- dojo/finding/queries.py | 5 +- dojo/finding/ui/views.py | 3 +- dojo/management/commands/dedupe.py | 9 +-- dojo/risk_acceptance/api/mixins.py | 9 ++- dojo/test/ui/views.py | 3 +- dojo/vulnerability_id/queries.py | 83 ++++++++++++++++++++++++++++ unittests/test_importers_importer.py | 13 ++--- unittests/test_vulnerability_id.py | 66 +++++++++++++++++++++- 13 files changed, 260 insertions(+), 63 deletions(-) diff --git a/dojo/apps.py b/dojo/apps.py index 7de69f92288..3ea65c281f5 100644 --- a/dojo/apps.py +++ b/dojo/apps.py @@ -113,6 +113,10 @@ def register_watson_models(app_config): watson.register(app_config.get_model("Location")) watson.register(app_config.get_model("Engagement"), fields=get_model_fields_with_extra(app_config.get_model("Engagement"), ("id", "product__name")), store=("product__name", )) watson.register(app_config.get_model("App_Analysis")) + # Watson stays on the legacy Vulnerability_Id table (unconditionally dual-written), so global + # search keeps working identically in both flag states with no buildwatson reindex. Moving the + # watson index (and the classic search view) to FindingVulnerabilityReference is a legacy-drop + # phase concern, not this reads-only flag flip. watson.register(app_config.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", )) # YourModel = app_config.get_model("YourModel") diff --git a/dojo/filters.py b/dojo/filters.py index f43562315e3..6422d66e997 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -52,10 +52,10 @@ Product, Product_Type, Test, - Vulnerability_Id, ) from dojo.product_type.queries import get_authorized_product_types from dojo.utils import get_system_setting, is_finding_groups_enabled, truncate_timezone_aware +from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids logger = logging.getLogger(__name__) @@ -73,17 +73,11 @@ def custom_filter(queryset, name, value): def custom_vulnerability_id_filter(queryset, name, value): values = value.split(",") - ids = Vulnerability_Id.objects \ - .filter(vulnerability_id__in=values) \ - .values_list("finding_id", flat=True) - return queryset.filter(id__in=ids) + return queryset.filter(id__in=finding_ids_with_vulnerability_ids(values, lookup="in")) def vulnerability_id_filter(queryset, name, value): - ids = Vulnerability_Id.objects \ - .filter(vulnerability_id=value) \ - .values_list("finding_id", flat=True) - return queryset.filter(id__in=ids) + return queryset.filter(id__in=finding_ids_with_vulnerability_ids(value, lookup="exact")) class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): diff --git a/dojo/finding/api/serializer.py b/dojo/finding/api/serializer.py index 8cbdf8507f7..49780338c86 100644 --- a/dojo/finding/api/serializer.py +++ b/dojo/finding/api/serializer.py @@ -301,6 +301,51 @@ class Meta: fields = ["vulnerability_id"] +@extend_schema_field(VulnerabilityIdSerializer(many=True)) +class VulnerabilityIdsField(serializers.Field): + + """ + Wire-frozen v2 vulnerability_ids field. + + Reads ``[{"vulnerability_id": str}]`` from the flag-appropriate store (legacy rows or the + entity references) — byte-identical either way because writes are dual. Accepts the same + object list (tolerating bare strings) on write and hands the parsed strings to create/update + under ``parsed_vulnerability_ids``, which funnel to save_vulnerability_ids (unchanged path). + """ + + def __init__(self, **kwargs): + kwargs["source"] = "*" + kwargs.setdefault("required", False) + super().__init__(**kwargs) + + def get_attribute(self, instance): + # source="*" -> to_representation receives the whole Finding. + return instance + + def to_representation(self, finding): + from dojo.vulnerability_id.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- flag seam + return [{"vulnerability_id": value} for value in finding_vulnerability_id_strings(finding)] + + def to_internal_value(self, data): + if not isinstance(data, list): + msg = "Expected a list of vulnerability ids." + raise serializers.ValidationError(msg) + parsed = [] + for item in data: + if isinstance(item, dict): + if "vulnerability_id" not in item: + msg = 'Each vulnerability id object requires a "vulnerability_id" key.' + raise serializers.ValidationError(msg) + parsed.append(item["vulnerability_id"]) + elif isinstance(item, str): + parsed.append(item) + else: + msg = 'Each vulnerability id must be a string or {"vulnerability_id": "..."}.' + raise serializers.ValidationError(msg) + # source="*" merges this dict into validated_data; create/update pop the key. + return {"parsed_vulnerability_ids": parsed} + + @extend_schema_field(serializers.CharField()) class CweField(serializers.Field): @@ -345,9 +390,7 @@ class FindingSerializer(serializers.ModelSerializer): finding_groups = FindingGroupSerializer( source="finding_group_set", many=True, read_only=True, ) - vulnerability_ids = VulnerabilityIdSerializer( - source="vulnerability_id_set", many=True, required=False, - ) + vulnerability_ids = VulnerabilityIdsField(required=False) cwes = FindingCweSerializer( source="finding_cwe_set", many=True, required=False, ) @@ -439,12 +482,10 @@ def update(self, instance, validated_data): # push_all_issues already checked in api views.py push_to_jira = validated_data.pop("push_to_jira") - # Save vulnerability ids and pop them - parsed_vulnerability_ids = [] - if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)): - logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids) - parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids) - logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0]) + # Save vulnerability ids and pop them (VulnerabilityIdsField parsed them to strings) + parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or [] + if parsed_vulnerability_ids: + logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0]) validated_data["cve"] = parsed_vulnerability_ids[0] # CWEs (mirror vulnerability_ids): the primary Finding.cwe plus extra Finding_CWE rows @@ -611,9 +652,7 @@ class FindingCreateSerializer(serializers.ModelSerializer): ) url = serializers.CharField(allow_null=True, default=None) push_to_jira = serializers.BooleanField(default=False) - vulnerability_ids = VulnerabilityIdSerializer( - source="vulnerability_id_set", many=True, required=False, - ) + vulnerability_ids = VulnerabilityIdsField(required=False) cwes = FindingCweSerializer( source="finding_cwe_set", many=True, required=False, ) @@ -647,15 +686,12 @@ def create(self, validated_data): notes = validated_data.pop("notes", None) found_by = validated_data.pop("found_by", None) reviewers = validated_data.pop("reviewers", None) - # Process the vulnerability IDs specially - parsed_vulnerability_ids = [] - if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)): - logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids) - parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids) - logger.debug("PARSED_VULNERABILITY_IDST: %s", parsed_vulnerability_ids) - logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0]) + # Process the vulnerability IDs specially (VulnerabilityIdsField parsed them to strings) + parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or [] + if parsed_vulnerability_ids: + logger.debug("PARSED_VULNERABILITY_IDS: %s", parsed_vulnerability_ids) + logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0]) validated_data["cve"] = parsed_vulnerability_ids[0] - # validated_data["unsaved_vulnerability_ids"] = parsed_vulnerability_ids # CWEs (mirror vulnerability_ids): the primary cwe plus extras. Precedence: an explicit # scalar `cwe` in the request stays the primary and every `cwes` entry is treated as an diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index 36d2a4a402e..882d1d720cc 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -9,6 +9,7 @@ from dojo.celery import app from dojo.models import Endpoint_Status, Finding, System_Settings +from dojo.vulnerability_id.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -340,11 +341,11 @@ def build_candidate_scope_queryset(test, mode="deduplication", service=None): queryset = Finding.objects.filter(scope_q) if settings.V3_FEATURE_LOCATIONS: - prefetch_list = ["locations__location__url", "vulnerability_id_set", "finding_cwe_set", "found_by"] + prefetch_list = ["locations__location__url", vulnerability_id_prefetch(), "finding_cwe_set", "found_by"] else: # TODO: Delete this after the move to Locations # Base prefetches for both modes - prefetch_list = ["endpoints", "vulnerability_id_set", "finding_cwe_set", "found_by"] + prefetch_list = ["endpoints", vulnerability_id_prefetch(), "finding_cwe_set", "found_by"] # Prefetch all endpoint statuses with their endpoint for reimport mode. # The non-special filtering (excluding false_positive, out_of_scope, risk_accepted) diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 03473091de2..42721500a56 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -705,9 +705,13 @@ def copy(self, test=None): copy.found_by.set(old_found_by) # Assign any tags copy.tags.set(old_tags) - # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util) - for vulnerability_id in self.vulnerability_id_set.all(): - Vulnerability_Id.objects.create(finding=copy, vulnerability_id=vulnerability_id.vulnerability_id) + # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util). + # Route vulnerability ids through the dual-write seam so the copy gets legacy rows AND + # entity references (copy.cve was already carried over by copy_model_util). + from dojo.vulnerability_id.manager import persist_for_finding # noqa: PLC0415 -- avoid import cycle + vulnerability_id_strings = [row.vulnerability_id for row in self.vulnerability_id_set.all()] + if vulnerability_id_strings: + persist_for_finding(copy, vulnerability_id_strings, delete_existing=False) for finding_cwe in self.finding_cwe_set.all(): Finding_CWE.objects.create(finding=copy, cwe=finding_cwe.cwe) @@ -838,12 +842,18 @@ def _get_unsaved_vulnerability_ids(finding) -> str: def _get_saved_vulnerability_ids(finding) -> str: if finding.id is not None: - # Use the reverse relation (vulnerability_id_set) rather than a fresh - # Vulnerability_Id.objects.filter(...) so prefetch_related("vulnerability_id_set") - # is honored — avoids an N+1 (COUNT + SELECT per finding) during dedupe/hashcode. - vulnerability_id_str_list = [str(vulnerability_id) for vulnerability_id in finding.vulnerability_id_set.all()] + from dojo.vulnerability_id.queries import use_entity_reads # noqa: PLC0415 + if use_entity_reads(): + # Entity store: the prefetch-honoring reverse relation (vulnerability_references + # + select_related("vulnerability")) so Prefetch is honored — no N+1 in dedupe. + vulnerability_id_str_list = [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] + else: + # Use the reverse relation (vulnerability_id_set) rather than a fresh + # Vulnerability_Id.objects.filter(...) so prefetch_related("vulnerability_id_set") + # is honored — avoids an N+1 (COUNT + SELECT per finding) during dedupe/hashcode. + vulnerability_id_str_list = [str(vulnerability_id) for vulnerability_id in finding.vulnerability_id_set.all()] deduplicationLogger.debug("get_vulnerability_ids after the finding was saved. Vulnerability references count: " + str(len(vulnerability_id_str_list))) - # sort vulnerability_ids strings + # sort vulnerability_ids strings (no dedupe — byte parity with the legacy store) return "".join(sorted(vulnerability_id_str_list)) return "" @@ -1383,9 +1393,14 @@ def get_references_with_links(self): @cached_property def vulnerability_ids(self): + from dojo.vulnerability_id.queries import use_entity_reads # noqa: PLC0415 # Get vulnerability ids from database and convert to list of strings - vulnerability_ids_model = self.vulnerability_id_set.all() - vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model] + if use_entity_reads(): + # Entity store: reverse relation is ordered by FindingVulnerabilityReference.Meta.ordering. + vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in self.vulnerability_references.all()] + else: + vulnerability_ids_model = self.vulnerability_id_set.all() + vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model] # Synchronize the cve field with the unsaved_vulnerability_ids # We do this to be as flexible as possible to handle the fields until diff --git a/dojo/finding/queries.py b/dojo/finding/queries.py index 82adda8773d..2d849ad56e7 100644 --- a/dojo/finding/queries.py +++ b/dojo/finding/queries.py @@ -20,6 +20,7 @@ def get_auth_filter(key): return None Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task +from dojo.vulnerability_id.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) @@ -111,7 +112,7 @@ def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=Tr "status_finding", "finding_group_set", "finding_group_set__jira_issue", # Include both variants - "vulnerability_id_set", + vulnerability_id_prefetch(), ) base_status = LocationFindingReference.objects.prefetch_related("location__url").all() prefetched_findings = prefetched_findings.annotate( @@ -147,7 +148,7 @@ def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=Tr "status_finding", "finding_group_set", "finding_group_set__jira_issue", # Include both variants - "vulnerability_id_set", + vulnerability_id_prefetch(), ) base_status = Endpoint_Status.objects.prefetch_related("endpoint") status = Case( diff --git a/dojo/finding/ui/views.py b/dojo/finding/ui/views.py index a0213630024..a1f73dc2e5c 100644 --- a/dojo/finding/ui/views.py +++ b/dojo/finding/ui/views.py @@ -121,6 +121,7 @@ reopen_external_issue, update_external_issue, ) +from dojo.vulnerability_id.queries import vulnerability_id_prefetch JFORM_PUSH_TO_JIRA_MESSAGE = "jform.push_to_jira: %s" @@ -168,7 +169,7 @@ def prefetch_for_similar_findings(findings): prefetched_findings = prefetched_findings.prefetch_related("notes") prefetched_findings = prefetched_findings.prefetch_related("tags") prefetched_findings = prefetched_findings.prefetch_related( - "vulnerability_id_set", + vulnerability_id_prefetch(), ) else: logger.debug("unable to prefetch because query was already executed") diff --git a/dojo/management/commands/dedupe.py b/dojo/management/commands/dedupe.py index 5e926dbad99..f2d9468705a 100644 --- a/dojo/management/commands/dedupe.py +++ b/dojo/management/commands/dedupe.py @@ -20,6 +20,7 @@ get_system_setting, mass_model_updater, ) +from dojo.vulnerability_id.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -97,10 +98,10 @@ def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedup "test", "test__engagement", "test__engagement__product", "test__test_type", ).prefetch_related( "locations", - # vulnerability_id_set feeds hash_code computation for parsers whose + # vulnerability id store feeds hash_code computation for parsers whose # HASHCODE_FIELDS_PER_SCANNER includes vulnerability_ids; prefetch to avoid # a per-finding query in get_vulnerability_ids(). - "vulnerability_id_set", + vulnerability_id_prefetch(), Prefetch( "original_finding", queryset=Finding.objects.only("id", "duplicate_finding_id").order_by("-id"), @@ -113,10 +114,10 @@ def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedup "test", "test__engagement", "test__engagement__product", "test__test_type", ).prefetch_related( "endpoints", - # vulnerability_id_set feeds hash_code computation for parsers whose + # vulnerability id store feeds hash_code computation for parsers whose # HASHCODE_FIELDS_PER_SCANNER includes vulnerability_ids; prefetch to avoid # a per-finding query in get_vulnerability_ids(). - "vulnerability_id_set", + vulnerability_id_prefetch(), Prefetch( "original_finding", queryset=Finding.objects.only("id", "duplicate_finding_id").order_by("-id"), diff --git a/dojo/risk_acceptance/api/mixins.py b/dojo/risk_acceptance/api/mixins.py index cc245943d74..e4c4dc6e0b2 100644 --- a/dojo/risk_acceptance/api/mixins.py +++ b/dojo/risk_acceptance/api/mixins.py @@ -12,8 +12,9 @@ from dojo.authorization.api_permissions import UserHasRiskAcceptanceRelatedObjectPermission from dojo.engagement.queries import get_authorized_engagements -from dojo.models import Engagement, Risk_Acceptance, User, Vulnerability_Id +from dojo.models import Engagement, Risk_Acceptance, User from dojo.risk_acceptance.api.serializer import RiskAcceptanceSerializer +from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids AcceptedRisk = NamedTuple("AcceptedRisk", (("vulnerability_id", str), ("justification", str), ("accepted_by", str))) @@ -90,10 +91,8 @@ def accept_risks(self, request): def _accept_risks(accepted_risks: list[AcceptedRisk], base_findings: QuerySet, owner: User): accepted = [] for risk in accepted_risks: - vulnerability_ids = Vulnerability_Id.objects \ - .filter(vulnerability_id=risk.vulnerability_id) \ - .values("finding") - findings = base_findings.filter(id__in=vulnerability_ids) + finding_ids = finding_ids_with_vulnerability_ids(risk.vulnerability_id, lookup="exact") + findings = base_findings.filter(id__in=finding_ids) if findings.exists(): # TODO: we could use risk.vulnerability_id to name the risk_acceptance, but would need to check for existing risk_acceptances in that case # so for now we add some timestamp based suffix diff --git a/dojo/test/ui/views.py b/dojo/test/ui/views.py index b549a1c2236..70d8d866fbf 100644 --- a/dojo/test/ui/views.py +++ b/dojo/test/ui/views.py @@ -82,6 +82,7 @@ process_tag_notifications, redirect_to_return_url_or_else, ) +from dojo.vulnerability_id.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) parse_logger = logging.getLogger("dojo") @@ -172,7 +173,7 @@ def get_initial_context(self, request: HttpRequest, test: Test): "jira_project": jira_services.get_project(test), "bulk_edit_form": FindingBulkUpdateForm(request.GET), "enable_table_filtering": get_system_setting("enable_ui_table_based_searching"), - "finding_groups": test.finding_group_set.all().prefetch_related("findings", "jira_issue", "creator", "findings__vulnerability_id_set"), + "finding_groups": test.finding_group_set.all().prefetch_related("findings", "jira_issue", "creator", vulnerability_id_prefetch(prefix="findings__")), "finding_group_by_options": Finding_Group.GROUP_BY_OPTIONS, } # Set the form using the context, and then update the context diff --git a/dojo/vulnerability_id/queries.py b/dojo/vulnerability_id/queries.py index 5fc97d7eede..0b31b07aeca 100644 --- a/dojo/vulnerability_id/queries.py +++ b/dojo/vulnerability_id/queries.py @@ -1,15 +1,98 @@ import logging +from django.conf import settings +from django.db.models import OuterRef, Prefetch + try: from dojo.authorization.query_filters import get_auth_filter except ImportError: def get_auth_filter(key): return None +from dojo.finding.models import Vulnerability_Id from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId logger = logging.getLogger(__name__) +# --------------------------------------------------------------------------- +# Read seam. The V3_FEATURE_VULNERABILITY_IDS flag is consulted ONLY here and in +# the two Finding methods (vulnerability_ids property, get_vulnerability_ids) — +# nowhere else. Writes are unconditionally dual (see manager.py), so both stores +# hold identical data and this flag selects the READ store only. +# --------------------------------------------------------------------------- + + +def use_entity_reads() -> bool: + """True when findings should read vulnerability ids from the entity/reference tables.""" + return getattr(settings, "V3_FEATURE_VULNERABILITY_IDS", False) + + +def finding_ids_with_vulnerability_ids(values, *, lookup="in"): + """ + QuerySet of finding ids whose vulnerability ids match ``values`` (``lookup`` = 'in' or 'exact'). + + Backs both filter functions in dojo/filters.py and the risk-acceptance exact-match lookup. + """ + if use_entity_reads(): + return ( + FindingVulnerabilityReference.objects + .filter(**{f"vulnerability__vulnerability_id__{lookup}": values}) + .values_list("finding_id", flat=True) + ) + return ( + Vulnerability_Id.objects + .filter(**{f"vulnerability_id__{lookup}": values}) + .values_list("finding_id", flat=True) + ) + + +def vulnerability_id_prefetch(prefix=""): + """ + Prefetch spec for a finding's vulnerability ids — a string (flag off) or a Prefetch (flag on). + + ``prefix`` supports nested lookups (e.g. ``"findings__"`` to prefetch through a finding group). + """ + if use_entity_reads(): + return Prefetch( + f"{prefix}vulnerability_references", + queryset=FindingVulnerabilityReference.objects.select_related("vulnerability"), + ) + return f"{prefix}vulnerability_id_set" + + +def finding_vulnerability_id_strings(finding): + """ + Ordered vulnerability id strings for a saved finding from the flag-appropriate store. + + The RAW relation (no cve merge, no dedupe) — backs the wire-frozen v2 serializer field, whose + output must stay byte-identical to reading the legacy vulnerability_id_set. + """ + if use_entity_reads(): + return [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] + return [row.vulnerability_id for row in finding.vulnerability_id_set.all()] + + +def first_vulnerability_id_subquery(): + """ + Subquery selecting a finding's primary (first) vulnerability id string, for annotation. + + Flag on: the reference at ``order == 0`` (== the finding's cve). Flag off: today's variant — + the lowest-PK legacy row. + """ + if use_entity_reads(): + return ( + FindingVulnerabilityReference.objects + .filter(finding=OuterRef("pk"), order=0) + .values("vulnerability__vulnerability_id")[:1] + ) + return ( + Vulnerability_Id.objects + .filter(finding=OuterRef("pk")) + .order_by("id") + .values("vulnerability_id")[:1] + ) + + def get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): impl = get_auth_filter("vulnerability_id.get_authorized_entities") if impl: diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py index 0b38f132c34..e883f6c6613 100644 --- a/unittests/test_importers_importer.py +++ b/unittests/test_importers_importer.py @@ -7,6 +7,7 @@ from rest_framework.authtoken.models import Token from rest_framework.test import APIClient +from dojo.finding.helper import save_vulnerability_ids from dojo.importers.default_importer import DefaultImporter from dojo.importers.default_reimporter import DefaultReImporter from dojo.models import ( @@ -1066,10 +1067,8 @@ def test_clear_vulnerability_ids_on_empty_list(self): finding.reporter = self.testuser finding.save() - # Add some vulnerability IDs - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-5678") - finding.cve = "CVE-2020-1234" + # Add some vulnerability IDs via the dual-write seam (legacy rows + entity references) + save_vulnerability_ids(finding, ["CVE-2020-1234", "CVE-2020-5678"]) finding.save() # Verify initial state @@ -1102,10 +1101,8 @@ def test_change_vulnerability_ids_on_reimport(self): finding.reporter = self.testuser finding.save() - # Add initial vulnerability IDs - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-5678") - finding.cve = "CVE-2020-1234" + # Add initial vulnerability IDs via the dual-write seam (legacy rows + entity references) + save_vulnerability_ids(finding, ["CVE-2020-1234", "CVE-2020-5678"]) finding.save() # Verify initial state diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index 5433fc64635..7f83c3a3c54 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -8,9 +8,10 @@ entity rows (orphans are retained). """ from django.contrib.auth import get_user_model -from django.test import TestCase +from django.test import TestCase, override_settings from django.utils.timezone import now +from dojo.finding.api.serializer import VulnerabilityIdsField from dojo.finding.helper import save_vulnerability_ids from dojo.models import ( Engagement, @@ -24,6 +25,7 @@ VulnerabilityId, ) from dojo.vulnerability_id.manager import VulnerabilityIdManager, persist_for_finding +from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids class VulnerabilityIdDualWriteTestCase(TestCase): @@ -124,3 +126,65 @@ def test_batch_record_and_reconcile_flush(self): with self.subTest("buffers cleared after flush"): self.assertEqual(manager.pending, []) self.assertEqual(manager.pending_deletes, []) + + +class TestReadSeamFlagParity(VulnerabilityIdDualWriteTestCase): + + """ + The read seam produces byte-identical results in both flag states for a seam-written finding + (dual-write keeps both stores in sync). Covers the v2 wire shape, the hash input + (get_vulnerability_ids), the vulnerability_ids property, and the filter seam. + """ + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + cls.finding = Finding.objects.create( + test=cls.test, reporter=cls.user, title="parity", severity="High", numerical_severity="S0", + ) + save_vulnerability_ids(cls.finding, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) + cls.finding.save() + + def _fresh(self): + # Fresh instance each read: vulnerability_ids is a cached_property. + return Finding.objects.get(pk=self.finding.pk) + + def test_v2_wire_shape_parity(self): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off = VulnerabilityIdsField().to_representation(self._fresh()) + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on = VulnerabilityIdsField().to_representation(self._fresh()) + self.assertEqual(off, on) + self.assertEqual( + off, + [{"vulnerability_id": "CVE-P-1"}, {"vulnerability_id": "CVE-P-2"}, {"vulnerability_id": "CVE-P-3"}], + ) + + def test_hash_input_parity(self): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off = self._fresh().get_vulnerability_ids() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on = self._fresh().get_vulnerability_ids() + self.assertEqual(off, on) + + def test_property_parity(self): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off = self._fresh().vulnerability_ids + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on = self._fresh().vulnerability_ids + self.assertEqual(off, on) + self.assertEqual(off, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) + + def test_filter_seam_parity(self): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off = set(finding_ids_with_vulnerability_ids(["CVE-P-2"], lookup="in")) + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on = set(finding_ids_with_vulnerability_ids(["CVE-P-2"], lookup="in")) + self.assertEqual(off, on) + self.assertEqual(off, {self.finding.pk}) + # Exact-match lane too. + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + self.assertEqual( + set(finding_ids_with_vulnerability_ids("CVE-P-1", lookup="exact")), + {self.finding.pk}, + ) From 932c8b534106f5e491be51e5139a709ed72bf75f Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:00:36 -0600 Subject: [PATCH 05/18] feat(vuln-id): backfill + reconciliation management commands - migrate_vulnerability_ids: wraps dojo.vulnerability_id.backfill.run_backfill (the same routine 0286 runs); --window-size; PostgreSQL-guarded; prints per-step row counts; idempotent/resumable so big tenants pre-run it before the deploy. - verify_vulnerability_ids: read-only reconciliation, nonzero exit on any drift so it can gate a ring flip. Per sampled finding (--sample, default 1000): every legacy (finding, id) pair has a reference; extra references allowed only when cve-only-derived (string == finding.cve); the order-0 reference == cve. Plus legacy-row / entity / reference / orphan-entity counts, both table names printed with roles. --json for machine output. Tests (fixture-free): migrate backfills legacy-only findings to entities/refs with order 0 == cve and is a no-op on re-run; verify passes on consistent data and raises SystemExit on a legacy row with no matching reference. Verified on the dev DB: migrate_vulnerability_ids runs clean and idempotent; verify_vulnerability_ids exits 0. Co-Authored-By: Claude Fable 5 --- .../commands/migrate_vulnerability_ids.py | 40 +++++++ .../commands/verify_vulnerability_ids.py | 113 ++++++++++++++++++ unittests/test_vulnerability_id.py | 45 +++++++ 3 files changed, 198 insertions(+) create mode 100644 dojo/management/commands/migrate_vulnerability_ids.py create mode 100644 dojo/management/commands/verify_vulnerability_ids.py diff --git a/dojo/management/commands/migrate_vulnerability_ids.py b/dojo/management/commands/migrate_vulnerability_ids.py new file mode 100644 index 00000000000..8027b52d4d1 --- /dev/null +++ b/dojo/management/commands/migrate_vulnerability_ids.py @@ -0,0 +1,40 @@ +import logging + +from django.core.management.base import BaseCommand +from django.db import connection + +from dojo.vulnerability_id.backfill import DEFAULT_WINDOW_SIZE, run_backfill + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + """ + Backfill the VulnerabilityId entity + FindingVulnerabilityReference tables from the legacy + dojo_vulnerability_id table and dojo_finding.cve. + + Idempotent and resumable (insert-only + ON CONFLICT DO NOTHING), so it is safe to run + repeatedly: large tenants can pre-run it on the previous release, then it is an instant + catch-up during the deploy-time migration. Shares dojo.vulnerability_id.backfill.run_backfill + with migration 0286. + """ + + help = "Idempotently backfill the vulnerability-id entity + reference tables (PostgreSQL only)." + + def add_arguments(self, parser): + parser.add_argument( + "--window-size", + type=int, + default=DEFAULT_WINDOW_SIZE, + help=f"finding_id window size for the reference build (default {DEFAULT_WINDOW_SIZE}).", + ) + + def handle(self, *args, **options): + if connection.vendor != "postgresql": + self.stderr.write(f"Vulnerability-id backfill is PostgreSQL only; nothing to do on {connection.vendor}.") + return + counts = run_backfill(logger=logger, window_size=options["window_size"]) + for step, count in counts.items(): + self.stdout.write(f"{step}: {count}") + self.stdout.write(self.style.SUCCESS("Vulnerability ID backfill complete.")) diff --git a/dojo/management/commands/verify_vulnerability_ids.py b/dojo/management/commands/verify_vulnerability_ids.py new file mode 100644 index 00000000000..4c106efd0b7 --- /dev/null +++ b/dojo/management/commands/verify_vulnerability_ids.py @@ -0,0 +1,113 @@ +import json +import logging + +from django.core.management.base import BaseCommand +from django.db.models import Q + +from dojo.models import Finding, FindingVulnerabilityReference, Vulnerability_Id, VulnerabilityId + +logger = logging.getLogger(__name__) + +# Both stores, with their roles — printed so operators know exactly what is being compared. +LEGACY_TABLE = "dojo_vulnerability_id" # legacy per-finding rows (authoritative for writes) +ENTITY_TABLE = "dojo_vulnerabilityid" # normalized VulnerabilityId registry +REFERENCE_TABLE = "dojo_findingvulnerabilityreference" # ordered Finding -> VulnerabilityId links + + +class Command(BaseCommand): + + """ + Read-only reconciliation of the legacy vulnerability-id store against the entity/reference + store. Exits nonzero on any discrepancy, so it can gate a ring flip. + + Checks per sampled finding: (1) every legacy (finding, id) pair has a matching reference; + extra references are tolerated ONLY when they are cve-only-derived (string == finding.cve); + (2) the order-0 reference string equals finding.cve wherever cve is set. Plus row / entity / + reference / orphan-entity counts. + """ + + help = "Verify the legacy vulnerability-id store matches the entity/reference store (nonzero exit on drift)." + + def add_arguments(self, parser): + parser.add_argument( + "--sample", + type=int, + default=1000, + help="Number of findings to spot-check for pair parity / order-0 (default 1000).", + ) + parser.add_argument("--json", action="store_true", dest="as_json", help="Emit the report as JSON.") + + def handle(self, *args, **options): + sample = options["sample"] + discrepancies = [] + + counts = { + f"legacy_rows ({LEGACY_TABLE})": Vulnerability_Id.objects.count(), + f"entities ({ENTITY_TABLE})": VulnerabilityId.objects.count(), + f"references ({REFERENCE_TABLE})": FindingVulnerabilityReference.objects.count(), + "orphan_entities": VulnerabilityId.objects.filter(finding_references__isnull=True).count(), + } + + sampled = ( + Finding.objects + .filter( + # Reverse relation query names: legacy FK -> "vulnerability_id", entity refs -> + # "vulnerability_references" (the "_set" form is the attribute accessor, not a lookup). + Q(vulnerability_id__isnull=False) + | Q(vulnerability_references__isnull=False) + | (Q(cve__isnull=False) & ~Q(cve="")), + ) + .distinct() + .order_by("id") + .prefetch_related("vulnerability_id_set", "vulnerability_references__vulnerability")[:sample] + ) + + checked = 0 + for finding in sampled: + checked += 1 + legacy_strings = {row.vulnerability_id for row in finding.vulnerability_id_set.all()} + references = list(finding.vulnerability_references.all()) + reference_strings = {ref.vulnerability.vulnerability_id for ref in references} + + missing = legacy_strings - reference_strings + if missing: + discrepancies.append(f"finding {finding.id}: legacy ids missing a reference: {sorted(missing)}") + + extras = reference_strings - legacy_strings + unexpected = {value for value in extras if value != finding.cve} + if unexpected: + discrepancies.append(f"finding {finding.id}: references with no legacy row (not cve-derived): {sorted(unexpected)}") + + if finding.cve: + primary = next((ref for ref in references if ref.order == 0), None) + primary_string = primary.vulnerability.vulnerability_id if primary else None + if primary_string != finding.cve: + discrepancies.append( + f"finding {finding.id}: order-0 reference {primary_string!r} != cve {finding.cve!r}", + ) + + report = { + "counts": counts, + "findings_checked": checked, + "discrepancies": discrepancies, + "ok": not discrepancies, + } + + if options["as_json"]: + self.stdout.write(json.dumps(report, indent=2)) + else: + self.stdout.write("Vulnerability ID store reconciliation") + for label, value in counts.items(): + self.stdout.write(f" {label}: {value}") + self.stdout.write(f" findings checked: {checked}") + if discrepancies: + self.stdout.write(self.style.ERROR(f" {len(discrepancies)} discrepancies:")) + for line in discrepancies: + self.stdout.write(self.style.ERROR(f" - {line}")) + else: + self.stdout.write(self.style.SUCCESS(" no discrepancies")) + + if discrepancies: + # Nonzero exit for CI / ring-flip gating. + msg = f"{len(discrepancies)} vulnerability-id reconciliation discrepancies" + raise SystemExit(msg) diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index 7f83c3a3c54..b7bebac3324 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -8,6 +8,7 @@ entity rows (orphans are retained). """ from django.contrib.auth import get_user_model +from django.core.management import call_command from django.test import TestCase, override_settings from django.utils.timezone import now @@ -188,3 +189,47 @@ def test_filter_seam_parity(self): set(finding_ids_with_vulnerability_ids("CVE-P-1", lookup="exact")), {self.finding.pk}, ) + + +class TestVulnerabilityIdCommands(VulnerabilityIdDualWriteTestCase): + + """ + migrate_vulnerability_ids backfills entities/references from legacy rows (idempotently); + verify_vulnerability_ids passes when the stores agree and exits nonzero on drift. + """ + + def test_migrate_command_backfills_idempotently(self): + finding = self._make_finding("cmd-migrate") + finding.cve = "CVE-CMD-1" + finding.save() + # Legacy rows WITHOUT entity references (bypassing the dual-write seam, i.e. pre-migration). + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-CMD-1") + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-CMD-2") + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) + + call_command("migrate_vulnerability_ids") + + # cve-match lands at order 0. + self.assertEqual(self._refs(finding), {0: "CVE-CMD-1", 1: "CVE-CMD-2"}) + self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-CMD-1").exists()) + + # Idempotent re-run: no change. + call_command("migrate_vulnerability_ids") + self.assertEqual(self._refs(finding), {0: "CVE-CMD-1", 1: "CVE-CMD-2"}) + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 2) + + def test_verify_command_passes_when_consistent(self): + finding = self._make_finding("cmd-verify-ok") + save_vulnerability_ids(finding, ["CVE-V-1", "CVE-V-2"]) + finding.save() + # Consistent stores => no SystemExit. + call_command("verify_vulnerability_ids") + + def test_verify_command_fails_on_discrepancy(self): + finding = self._make_finding("cmd-verify-bad") + finding.cve = "CVE-BAD-1" + finding.save() + # Legacy row with no matching entity reference => a discrepancy. + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-BAD-1") + with self.assertRaises(SystemExit): + call_command("verify_vulnerability_ids") From b97fc306b8d89c2cca1970845e7817298eb1a314 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:06:56 -0600 Subject: [PATCH 06/18] test(vulnerability_id): add upgrade read-parity tests (legacy rows -> backfill -> flag parity) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TestReadSeamFlagParity seeds via the dual-write seam, so both stores are trivially in sync. These new tests cover the actual UPGRADE scenario a deployment hits: legacy Vulnerability_Id rows written the old way (no entity refs), then backfilled by migrate_vulnerability_ids, then read flag-off (legacy) vs flag-on (entity). - test_backfilled_normal_data_reads_identically: for seam-shaped data (cve == first/ lowest-PK row) get_vulnerability_ids, the vulnerability_ids property, and the raw v2 wire shape are byte-identical across the flag. - test_backfilled_cve_not_first_row_preserves_hash_and_property: anomalous data where cve is not the first-created legacy row keeps hash + property byte-identical (both are order-insensitive by construction); only the RAW v2 wire ORDER differs (legacy PK-asc vs entity cve-first, same set) — asserted so the bounded divergence is deliberate. Co-Authored-By: Claude Opus 4.8 (1M context) --- unittests/test_vulnerability_id.py | 85 ++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index b7bebac3324..8d73f5cc63c 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -233,3 +233,88 @@ def test_verify_command_fails_on_discrepancy(self): Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-BAD-1") with self.assertRaises(SystemExit): call_command("verify_vulnerability_ids") + + +class TestUpgradeReadParity(VulnerabilityIdDualWriteTestCase): + + """ + Upgrade path: legacy Vulnerability_Id rows written the OLD way (before the entity existed) and + then backfilled by migrate_vulnerability_ids must read back IDENTICALLY in both flag states. + + Distinct from TestReadSeamFlagParity, which seeds through the dual-write seam (so both stores + are trivially in sync). Here only the LEGACY rows are written — the true pre-upgrade state — + then backfilled, then flag-off (legacy read) is compared against flag-on (entity read). This is + the scenario a real deployment actually hits when the flag is first turned on after a migration. + """ + + def _seed_legacy_only(self, finding, cve, ordered_ids): + """ + Write legacy rows directly (creation order == PK order), set the primary cve, and assert + the pre-migration state (no entity references yet) — i.e. exactly what pre-upgrade data looks + like. + """ + finding.cve = cve + finding.save() + for vid in ordered_ids: + Vulnerability_Id.objects.create(finding=finding, vulnerability_id=vid) + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) + + def _fresh(self, pk): + # Fresh instance per read: vulnerability_ids is a cached_property. + return Finding.objects.get(pk=pk) + + def _reads(self, pk): + return ( + self._fresh(pk).get_vulnerability_ids(), + self._fresh(pk).vulnerability_ids, + [r["vulnerability_id"] for r in VulnerabilityIdsField().to_representation(self._fresh(pk))], + ) + + def test_backfilled_normal_data_reads_identically(self): + # Seam/historical invariant: the primary cve IS the first-created (lowest-PK) legacy row. + finding = self._make_finding("upgrade-normal") + self._seed_legacy_only(finding, cve="CVE-U-1", ordered_ids=["CVE-U-1", "CVE-U-2", "CVE-U-3"]) + + call_command("migrate_vulnerability_ids") + self.assertEqual(self._refs(finding)[0], "CVE-U-1") # cve at order 0 + + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off_hash, off_prop, off_wire = self._reads(finding.pk) + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on_hash, on_prop, on_wire = self._reads(finding.pk) + + # Full parity: hash input, the merged property, AND the raw v2 wire order. + self.assertEqual(off_hash, on_hash) + self.assertEqual(off_prop, on_prop) + self.assertEqual(off_wire, on_wire) + self.assertEqual(off_prop, ["CVE-U-1", "CVE-U-2", "CVE-U-3"]) + self.assertEqual(off_wire, ["CVE-U-1", "CVE-U-2", "CVE-U-3"]) + + def test_backfilled_cve_not_first_row_preserves_hash_and_property(self): + # Anomalous pre-upgrade data: the primary cve is NOT the first-created legacy row (e.g. the + # cve field was edited independently of the id rows historically). The hash (sorted) and the + # vulnerability_ids property (cve-merged + deduped) are order-insensitive by construction and + # stay byte-identical across flag states. Only the RAW v2 wire ORDER differs — legacy reads + # PK-ascending, the entity reads cve-first — carrying the SAME set in a different order. This + # divergence is bounded to this anomalous shape (never produced by save_vulnerability_ids, + # which always makes cve == ids[0] == the lowest-PK row) and is asserted here so any future + # change to it is deliberate, not silent. + finding = self._make_finding("upgrade-cve-mid") + self._seed_legacy_only(finding, cve="CVE-M-3", ordered_ids=["CVE-M-1", "CVE-M-2", "CVE-M-3"]) + + call_command("migrate_vulnerability_ids") + self.assertEqual(self._refs(finding)[0], "CVE-M-3") # cve at order 0 despite being last-created + + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off_hash, off_prop, off_wire = self._reads(finding.pk) + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on_hash, on_prop, on_wire = self._reads(finding.pk) + + # Hash + property: byte-identical across the flag. + self.assertEqual(off_hash, on_hash) + self.assertEqual(off_prop, on_prop) + self.assertEqual(off_prop, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # cve first, then PK-asc rest + # Raw v2 wire: same set, different order (the documented, bounded divergence). + self.assertEqual(off_wire, ["CVE-M-1", "CVE-M-2", "CVE-M-3"]) # legacy PK order + self.assertEqual(on_wire, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # entity cve-first order + self.assertEqual(set(off_wire), set(on_wire)) From bc0c7fc300772900c4196298d10908a92b8af737 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:44:36 -0600 Subject: [PATCH 07/18] test(vuln-id): re-baseline importer perf query counts for dual-write The unconditional dual-write of VulnerabilityId entities + FindingVulnerabilityReference rows adds a small constant per-import query delta (batched, not per-finding). Counts updated to match a pure-OSS CI run: dedup first_import +6 / second_import +4; reimport import1 +6 / reimport1 +12 / reimport2 +5, across TestDojoImporterPerformanceSmall and ...SmallLocations. Async-task counts unchanged (dual-write is synchronous). reimport3 (queries4) unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) --- unittests/test_importers_performance.py | 56 ++++++++++++------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 14fea501e17..eedc01943f9 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -343,11 +343,11 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=157, + expected_num_queries1=163, expected_num_async_tasks1=2, - expected_num_queries2=124, + expected_num_queries2=136, expected_num_async_tasks2=1, - expected_num_queries3=30, + expected_num_queries3=35, expected_num_async_tasks3=1, expected_num_queries4=106, expected_num_async_tasks4=0, @@ -367,11 +367,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=176, + expected_num_queries1=182, expected_num_async_tasks1=2, - expected_num_queries2=134, + expected_num_queries2=146, expected_num_async_tasks2=1, - expected_num_queries3=40, + expected_num_queries3=45, expected_num_async_tasks3=1, expected_num_queries4=106, expected_num_async_tasks4=0, @@ -392,11 +392,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=186, + expected_num_queries1=192, expected_num_async_tasks1=5, - expected_num_queries2=144, + expected_num_queries2=156, expected_num_async_tasks2=4, - expected_num_queries3=49, + expected_num_queries3=54, expected_num_async_tasks3=3, expected_num_queries4=115, expected_num_async_tasks4=3, @@ -526,9 +526,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=93, + expected_num_queries1=99, expected_num_async_tasks1=2, - expected_num_queries2=73, + expected_num_queries2=77, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -547,9 +547,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=112, + expected_num_queries1=118, expected_num_async_tasks1=2, - expected_num_queries2=93, + expected_num_queries2=97, expected_num_async_tasks2=2, ) @@ -580,9 +580,9 @@ def test_deduplication_performance_pghistory_async_wait(self): # returns instantly without executing dedup on the request's DB connection. with patch("celery.result.AsyncResult.get", return_value=None): self._deduplication_performance( - expected_num_queries1=94, + expected_num_queries1=100, expected_num_async_tasks1=2, - expected_num_queries2=74, + expected_num_queries2=78, expected_num_async_tasks2=2, dedup_mode="async_wait", check_duplicates=False, @@ -670,11 +670,11 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=164, + expected_num_queries1=170, expected_num_async_tasks1=2, - expected_num_queries2=133, + expected_num_queries2=145, expected_num_async_tasks2=1, - expected_num_queries3=38, + expected_num_queries3=43, expected_num_async_tasks3=1, expected_num_queries4=107, expected_num_async_tasks4=0, @@ -694,11 +694,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=185, + expected_num_queries1=191, expected_num_async_tasks1=2, - expected_num_queries2=145, + expected_num_queries2=157, expected_num_async_tasks2=1, - expected_num_queries3=50, + expected_num_queries3=55, expected_num_async_tasks3=1, expected_num_queries4=107, expected_num_async_tasks4=0, @@ -719,11 +719,11 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=198, + expected_num_queries1=204, expected_num_async_tasks1=5, - expected_num_queries2=158, + expected_num_queries2=170, expected_num_async_tasks2=4, - expected_num_queries3=59, + expected_num_queries3=64, expected_num_async_tasks3=3, expected_num_queries4=119, expected_num_async_tasks4=3, @@ -826,9 +826,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=100, + expected_num_queries1=106, expected_num_async_tasks1=2, - expected_num_queries2=76, + expected_num_queries2=80, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -846,8 +846,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=121, + expected_num_queries1=127, expected_num_async_tasks1=2, - expected_num_queries2=204, + expected_num_queries2=208, expected_num_async_tasks2=2, ) From d1e9573ee52dd834dc33feac37b9e4ae440ba0f8 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:38:15 -0600 Subject: [PATCH 08/18] test(vuln-id): fix fixture/perf tests for default-on entity reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three test-only fixes surfaced by V3_FEATURE_VULNERABILITY_IDS defaulting True in OSS (reads hit the entity store): - test_bulk_risk_acceptance_api + test_pdf_report_rendering seeded vuln ids via the legacy Vulnerability_Id table directly, so entity reads returned empty flag-on. Seed via save_vulnerability_ids (unconditional dual-write) instead — production code is unchanged and correct. bulk_risk verified locally (3 tests green flag-on); the pdf report test loads dojo_testdata (can't run in the Pro container) and relies on OSS CI. - test_tag_inheritance_perf: re-baseline the 6 EXPECTED_ZAP_* assertNumQueries constants for the dual-write delta (+2 import / +12 reimport-no-change / +6 reimport-with-new), from a pure-OSS CI run. Co-Authored-By: Claude Opus 4.8 (1M context) --- unittests/test_bulk_risk_acceptance_api.py | 16 ++++++++-------- unittests/test_pdf_report_rendering.py | 6 ++++-- unittests/test_tag_inheritance_perf.py | 16 ++++++++++------ 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/unittests/test_bulk_risk_acceptance_api.py b/unittests/test_bulk_risk_acceptance_api.py index 3d2847a3f5a..8f3ad086d87 100644 --- a/unittests/test_bulk_risk_acceptance_api.py +++ b/unittests/test_bulk_risk_acceptance_api.py @@ -9,6 +9,7 @@ Role, ) from dojo.authorization.roles_permissions import Roles +from dojo.finding.helper import save_vulnerability_ids from dojo.models import ( Dojo_User, Engagement, @@ -19,7 +20,6 @@ Test, Test_Type, User, - Vulnerability_Id, ) @@ -61,24 +61,24 @@ def create_finding(test: Test, reporter: User, cve: str) -> Finding: Finding.objects.bulk_create( create_finding(cls.test_a, cls.user, f"CVE-1999-{i}") for i in range(50, 150, 3)) for finding in Finding.objects.filter(test=cls.test_a): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_b, cls.user, f"CVE-1999-{i}") for i in range(51, 150, 3)) for finding in Finding.objects.filter(test=cls.test_b): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_c, cls.user, f"CVE-1999-{i}") for i in range(52, 150, 3)) for finding in Finding.objects.filter(test=cls.test_c): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_d, cls.user, f"CVE-2000-{i}") for i in range(50, 150, 3)) for finding in Finding.objects.filter(test=cls.test_d): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_e, cls.user, f"CVE-1999-{i}") for i in range(50, 150, 3)) for finding in Finding.objects.filter(test=cls.test_e): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) def setUp(self) -> None: self.client = APIClient() @@ -204,13 +204,13 @@ def create_finding(test, reporter, cve): Finding.objects.bulk_create( create_finding(cls.test_enabled, cls.writer, f"CVE-2024-{i}") for i in range(10)) for f in Finding.objects.filter(test=cls.test_enabled): - Vulnerability_Id.objects.get_or_create(finding=f, vulnerability_id=f.cve) + save_vulnerability_ids(f, [f.cve]) # Findings on the disabled product Finding.objects.bulk_create( create_finding(cls.test_disabled, cls.writer, f"CVE-2024-{i + 100}") for i in range(5)) for f in Finding.objects.filter(test=cls.test_disabled): - Vulnerability_Id.objects.get_or_create(finding=f, vulnerability_id=f.cve) + save_vulnerability_ids(f, [f.cve]) def _client_for(self, token): client = APIClient() diff --git a/unittests/test_pdf_report_rendering.py b/unittests/test_pdf_report_rendering.py index c363e3ea301..5449bced279 100644 --- a/unittests/test_pdf_report_rendering.py +++ b/unittests/test_pdf_report_rendering.py @@ -1,6 +1,7 @@ from django.template import engines from django.utils.timezone import now +from dojo.finding.helper import save_vulnerability_ids from dojo.models import ( Engagement, Finding, @@ -9,7 +10,6 @@ Test, Test_Type, User, - Vulnerability_Id, ) from unittests.dojo_test_case import DojoTestCase, versioned_fixtures @@ -176,7 +176,9 @@ def test_report_field_contains_rendered_content(self): def test_report_finding_table_includes_vulnerability_ids(self): """Finding PDF reports should show vulnerability IDs in the finding table.""" finding = self._create_finding() - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2026-12345") + # Dual-write (legacy row + entity reference) so the report's flag-appropriate + # read resolves whether V3_FEATURE_VULNERABILITY_IDS is on (default) or off. + save_vulnerability_ids(finding, ["CVE-2026-12345"]) html = self._render_finding_report(Finding.objects.filter(pk=finding.pk)) diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 5b3f07e8b59..9594e1dfeec 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -597,9 +597,13 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # Multiple-CWEs feature: +2 import / +2 reimport-no-change (Finding_CWE # store + bulk flush) and +10 reimport-with-new (per-finding reconcile reads # existing Finding_CWE rows for each changed finding). - EXPECTED_ZAP_IMPORT_V2 = 294 - EXPECTED_ZAP_IMPORT_V3 = 318 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 79 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 91 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 161 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 190 + # Vulnerability_Id entity dual-write (unconditional): +2 import / +12 + # reimport-no-change / +6 reimport-with-new queries (VulnerabilityId entity + + # FindingVulnerabilityReference bulk writes alongside the legacy rows; batched, + # not per-finding). + EXPECTED_ZAP_IMPORT_V2 = 296 + EXPECTED_ZAP_IMPORT_V3 = 320 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 91 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 103 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 167 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 196 From da41855de48cfcd0c84e200854141e541a2b49d7 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Wed, 22 Jul 2026 16:37:44 -0600 Subject: [PATCH 09/18] test(vuln-id): make setUpTestData valid under full model validation The rest-framework CI matrix runs a V3_FEATURE_LOCATIONS=True leg where base save() calls full_clean(). The new vuln-id test classes created Product/Test without the required description / scan_type / environment, so all 7 setUpClass hooks errored with ValidationError under that leg (they passed the False leg). Provide the required fields (Finding creation already skips validation via skip_validation=True, so it's unaffected). Reproduced locally with V3_FEATURE_LOCATIONS=True: 14 tests green. Co-Authored-By: Claude Opus 4.8 (1M context) --- unittests/test_migrations.py | 8 +++++++- unittests/test_vulnerability_id.py | 11 +++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/unittests/test_migrations.py b/unittests/test_migrations.py index 42e2cf9b98e..d476a4bac97 100644 --- a/unittests/test_migrations.py +++ b/unittests/test_migrations.py @@ -8,6 +8,7 @@ from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.models import ( + Development_Environment, Engagement, Finding, FindingVulnerabilityReference, @@ -198,12 +199,17 @@ class TestVulnerabilityIdBackfill(TestCase): def setUpTestData(cls): now = timezone.now() prod_type = Product_Type.objects.create(name="vulnid-pt") - product = Product.objects.create(prod_type=prod_type, name="vulnid-prod") + # description / scan_type / environment are required under full model validation + # (base save full_clean when V3_FEATURE_LOCATIONS is on — the CI "(true)" matrix leg). + product = Product.objects.create(prod_type=prod_type, name="vulnid-prod", description="vuln-id backfill test") engagement = Engagement.objects.create(product=product, target_start=now, target_end=now) test_type = Test_Type.objects.create(name="vulnid-tt") + environment = Development_Environment.objects.get_or_create(name="Development")[0] test = Test.objects.create( engagement=engagement, test_type=test_type, + scan_type="vulnid-tt", + environment=environment, target_start=now, target_end=now, ) diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index 8d73f5cc63c..bb892df08ac 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -15,6 +15,7 @@ from dojo.finding.api.serializer import VulnerabilityIdsField from dojo.finding.helper import save_vulnerability_ids from dojo.models import ( + Development_Environment, Engagement, Finding, FindingVulnerabilityReference, @@ -36,11 +37,17 @@ class VulnerabilityIdDualWriteTestCase(TestCase): @classmethod def setUpTestData(cls): cls.prod_type = Product_Type.objects.create(name="vulnid-dw-pt") - cls.product = Product.objects.create(prod_type=cls.prod_type, name="vulnid-dw-prod") + # description / scan_type / environment are required under full model validation + # (base save full_clean when V3_FEATURE_LOCATIONS is on — the CI "(true)" matrix leg). + cls.product = Product.objects.create( + prod_type=cls.prod_type, name="vulnid-dw-prod", description="vuln-id dual-write tests", + ) cls.engagement = Engagement.objects.create(product=cls.product, target_start=now(), target_end=now()) cls.test_type = Test_Type.objects.create(name="vulnid-dw-tt") + cls.environment = Development_Environment.objects.get_or_create(name="Development")[0] cls.test = Test.objects.create( - engagement=cls.engagement, test_type=cls.test_type, target_start=now(), target_end=now(), + engagement=cls.engagement, test_type=cls.test_type, scan_type="vulnid-dw-tt", + environment=cls.environment, target_start=now(), target_end=now(), ) cls.user = get_user_model().objects.create(username="vulnid-dw-user") From 6b369a7d6c84a7161d37dbd7303f466baa8fcb4d Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 23 Jul 2026 17:48:05 -0600 Subject: [PATCH 10/18] feat(vuln-id): rename to Vulnerability, fix cve-only + reimporter bugs, add cutover-safety tests Review response (DefectDojo/django-DefectDojo#15331) plus prep for the one-way legacy retirement. Rename (per review): VulnerabilityId -> Vulnerability; module dojo/vulnerability_id/ -> dojo/vulnerability/; table dojo_vulnerabilityid -> dojo_vulnerability (migration 0285 amended in place). String column stays vulnerability_id; reference FK stays `vulnerability`. Added vocabulary + porting-trap docstrings. Must-fixes: - MF4: UniqueConstraint(finding, order) on FindingVulnerabilityReference (model + migration 0285). - MF2: verify_vulnerability_ids downgrades order-0 != cve to a non-fatal warning when cve is not among the finding's ids (unreachable without breaking hash parity); stays a hard gate for genuine ordering drift. Bug fixes surfaced by the new tests: - cve-only hash-parity break: backfill step 4 fabricated a cve-only reference the live dual-write never creates, so entity reads diverged from legacy (dedup churn on deploy). Removed; the reference set now mirrors the legacy row set exactly. cve stays an orphan registry entity. - reimporter N+1: reconcile read the non-prefetched legacy relation under flag-on. Now reads through the seam (matches the prefetch). Perf re-baselined: -5 queries per reimport across 24 assertions. Reader ports toward the cutover (route through the flag seam, reversible): Finding.copy, reimporter reconcile. Tests (fixture-free, both flag states): legacy-independence (poison), dedup differential, real-scanner corpus (8 parsers), enumerable shapes (cve-only/null-cve/unicode/quotes/copy), MF2 warning, MF4 constraint. Co-Authored-By: Claude Opus 4.8 (1M context) --- dojo/api_v2/prefetch/registrations.py | 8 +- dojo/authorization/query_registrations.py | 4 +- .../0285_vulnerability_id_entity_tables.py | 50 +- ...0286_backfill_vulnerability_id_entities.py | 7 +- dojo/filters.py | 2 +- dojo/finding/api/serializer.py | 2 +- dojo/finding/deduplication.py | 2 +- dojo/finding/helper.py | 2 +- dojo/finding/models.py | 14 +- dojo/finding/queries.py | 2 +- dojo/finding/ui/views.py | 2 +- dojo/importers/base_importer.py | 2 +- dojo/importers/default_reimporter.py | 9 +- dojo/management/commands/dedupe.py | 2 +- .../commands/migrate_vulnerability_ids.py | 6 +- .../commands/verify_vulnerability_ids.py | 52 +- dojo/models.py | 4 +- dojo/risk_acceptance/api/mixins.py | 2 +- dojo/settings/settings.dist.py | 2 +- dojo/test/ui/views.py | 2 +- dojo/vulnerability/__init__.py | 1 + .../admin.py | 6 +- .../backfill.py | 50 +- .../manager.py | 25 +- dojo/vulnerability/models.py | 86 ++++ .../queries.py | 38 +- dojo/vulnerability_id/__init__.py | 1 - dojo/vulnerability_id/models.py | 71 --- unittests/test_migrations.py | 57 ++- unittests/test_tag_inheritance_perf.py | 2 +- unittests/test_vulnerability_id.py | 454 +++++++++++++++++- 31 files changed, 718 insertions(+), 249 deletions(-) create mode 100644 dojo/vulnerability/__init__.py rename dojo/{vulnerability_id => vulnerability}/admin.py (58%) rename dojo/{vulnerability_id => vulnerability}/backfill.py (70%) rename dojo/{vulnerability_id => vulnerability}/manager.py (85%) create mode 100644 dojo/vulnerability/models.py rename dojo/{vulnerability_id => vulnerability}/queries.py (77%) delete mode 100644 dojo/vulnerability_id/__init__.py delete mode 100644 dojo/vulnerability_id/models.py diff --git a/dojo/api_v2/prefetch/registrations.py b/dojo/api_v2/prefetch/registrations.py index e9922086471..8dc4faad7fb 100644 --- a/dojo/api_v2/prefetch/registrations.py +++ b/dojo/api_v2/prefetch/registrations.py @@ -110,11 +110,11 @@ from dojo.test.queries import get_authorized_test_imports, get_authorized_tests from dojo.tool_product.queries import get_authorized_tool_product_settings from dojo.url.models import URL -from dojo.vulnerability_id.models import ( +from dojo.vulnerability.models import ( FindingVulnerabilityReference, - VulnerabilityId, + Vulnerability, ) -from dojo.vulnerability_id.queries import ( +from dojo.vulnerability.queries import ( get_authorized_finding_vulnerability_references, get_authorized_vulnerability_id_entities, ) @@ -224,7 +224,7 @@ for model, helper in ( (Finding_Group, get_authorized_finding_groups), (Vulnerability_Id, get_authorized_vulnerability_ids), - (VulnerabilityId, get_authorized_vulnerability_id_entities), + (Vulnerability, get_authorized_vulnerability_id_entities), (FindingVulnerabilityReference, get_authorized_finding_vulnerability_references), ): register(model, discard_user(helper), "view") diff --git a/dojo/authorization/query_registrations.py b/dojo/authorization/query_registrations.py index b808dc50092..420cabdc60c 100644 --- a/dojo/authorization/query_registrations.py +++ b/dojo/authorization/query_registrations.py @@ -41,7 +41,7 @@ Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task -from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability def _resolve_user(user): @@ -421,7 +421,7 @@ def _get_authorized_vulnerability_ids(permission, queryset=None, user=None): def _get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): user = _resolve_user(user) - qs = queryset if queryset is not None else VulnerabilityId.objects.all() + qs = queryset if queryset is not None else Vulnerability.objects.all() if user is None or getattr(user, "is_anonymous", False): return qs.none() if _is_unrestricted(user, permission_to_action(permission)): diff --git a/dojo/db_migrations/0285_vulnerability_id_entity_tables.py b/dojo/db_migrations/0285_vulnerability_id_entity_tables.py index 43707fd979d..35b1c34b4d4 100644 --- a/dojo/db_migrations/0285_vulnerability_id_entity_tables.py +++ b/dojo/db_migrations/0285_vulnerability_id_entity_tables.py @@ -1,11 +1,11 @@ -# Vulnerability ID entity tables: the normalized VulnerabilityId registry and the ordered -# Finding -> VulnerabilityId reference table. Both tables are CREATED EMPTY here, so every index +# Vulnerability ID entity tables: the normalized Vulnerability registry and the ordered +# Finding -> Vulnerability reference table. Both tables are CREATED EMPTY here, so every index # (including the two GIN indexes) is built on zero rows — the DDL is effectively instant and needs # no AddIndexConcurrently. The data backfill runs separately in 0286 (atomic=False, resumable). # # >50M-row fallback (documented, NOT the default path): on installs where the subsequent 0286 # backfill would insert tens of millions of reference rows, GIN maintenance during the backfill -# dominates. Such installs can strip the two GinIndex entries from the VulnerabilityId Meta here, +# dominates. Such installs can strip the two GinIndex entries from the Vulnerability Meta here, # let 0286 populate the tables, then add both GIN indexes with CREATE INDEX CONCURRENTLY in a # follow-up migration. Not needed for typical installs. @@ -20,41 +20,41 @@ class Migration(migrations.Migration): dependencies = [ - ('dojo', '0284_backfill_finding_cwe'), + ("dojo", "0284_backfill_finding_cwe"), ] operations = [ migrations.CreateModel( - name='VulnerabilityId', + name="Vulnerability", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('vulnerability_id', models.TextField(unique=True, verbose_name='Vulnerability Id')), - ('vulnerability_id_type', models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True)), - ('epss_score', models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), - ('epss_percentile', models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), - ('known_exploited', models.BooleanField(blank=True, default=None, null=True)), - ('ransomware_used', models.BooleanField(blank=True, default=None, null=True)), - ('kev_date', models.DateField(blank=True, default=None, null=True)), - ('created', models.DateTimeField(auto_now_add=True)), - ('updated', models.DateTimeField(auto_now=True)), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("vulnerability_id", models.TextField(unique=True, verbose_name="Vulnerability Id")), + ("vulnerability_id_type", models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True)), + ("epss_score", models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), + ("epss_percentile", models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), + ("known_exploited", models.BooleanField(blank=True, default=None, null=True)), + ("ransomware_used", models.BooleanField(blank=True, default=None, null=True)), + ("kev_date", models.DateField(blank=True, default=None, null=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), ], options={ - 'indexes': [django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector('vulnerability_id', config='english', weight='A'), name='dojo_vulnid_entity_fts_gin'), django.contrib.postgres.indexes.GinIndex(fields=['vulnerability_id'], name='dojo_vulnid_entity_trgm', opclasses=['gin_trgm_ops']), models.Index(django.db.models.functions.text.Upper('vulnerability_id'), name='dojo_vulnid_entity_upper')], + "indexes": [django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector("vulnerability_id", config="english", weight="A"), name="dojo_vulnid_entity_fts_gin"), django.contrib.postgres.indexes.GinIndex(fields=["vulnerability_id"], name="dojo_vulnid_entity_trgm", opclasses=["gin_trgm_ops"]), models.Index(django.db.models.functions.text.Upper("vulnerability_id"), name="dojo_vulnid_entity_upper")], }, ), migrations.CreateModel( - name='FindingVulnerabilityReference', + name="FindingVulnerabilityReference", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('order', models.PositiveSmallIntegerField(default=0, help_text="Position in the finding's id list; 0 is the primary identifier.")), - ('created', models.DateTimeField(auto_now_add=True)), - ('finding', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='vulnerability_references', to='dojo.finding')), - ('vulnerability', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='finding_references', to='dojo.vulnerabilityid')), + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("order", models.PositiveSmallIntegerField(default=0, help_text="Position in the finding's id list; 0 is the primary identifier.")), + ("created", models.DateTimeField(auto_now_add=True)), + ("finding", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name="vulnerability_references", to="dojo.finding")), + ("vulnerability", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name="finding_references", to="dojo.vulnerability")), ], options={ - 'ordering': ['order'], - 'indexes': [models.Index(fields=['finding', 'order'], name='dojo_findin_finding_d1a7a8_idx'), models.Index(fields=['vulnerability'], name='dojo_findin_vulnera_13169c_idx')], - 'constraints': [models.UniqueConstraint(fields=('finding', 'vulnerability'), name='unique_finding_vulnerability')], + "ordering": ["order"], + "indexes": [models.Index(fields=["finding", "order"], name="dojo_findin_finding_d1a7a8_idx"), models.Index(fields=["vulnerability"], name="dojo_findin_vulnera_13169c_idx")], + "constraints": [models.UniqueConstraint(fields=("finding", "vulnerability"), name="unique_finding_vulnerability"), models.UniqueConstraint(fields=("finding", "order"), name="unique_finding_order")], }, ), ] diff --git a/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py b/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py index 0b1e0c4a067..6da497ca02d 100644 --- a/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py +++ b/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py @@ -1,11 +1,11 @@ -# Data-only backfill of the VulnerabilityId entity + FindingVulnerabilityReference tables created +# Data-only backfill of the Vulnerability entity + FindingVulnerabilityReference tables created # in 0285. atomic=False so each set-based statement (and each finding_id window in the reference # build) commits on its own — a crash mid-backfill resumes where it left off, and ON CONFLICT DO # NOTHING everywhere makes any re-run a no-op on completed work. Insert-only into the two NEW # tables: dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted, so # concurrent readers/writers are not blocked. # -# The actual work lives in dojo.vulnerability_id.backfill.run_backfill, shared with the +# The actual work lives in dojo.vulnerability.backfill.run_backfill, shared with the # `migrate_vulnerability_ids` management command (big installs pre-run the command on the previous # release; this migration is then an idempotent catch-up during the deploy). import logging @@ -23,7 +23,8 @@ def backfill_vulnerability_id_entities(apps, schema_editor): connection.vendor, ) return - from dojo.vulnerability_id.backfill import run_backfill # noqa: PLC0415 -- keep entity import lazy/out of migration graph build + from dojo.vulnerability.backfill import run_backfill # noqa: PLC0415 -- keep entity import lazy/out of migration graph build + run_backfill(logger=logger) diff --git a/dojo/filters.py b/dojo/filters.py index 6422d66e997..48f609c0b05 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -55,7 +55,7 @@ ) from dojo.product_type.queries import get_authorized_product_types from dojo.utils import get_system_setting, is_finding_groups_enabled, truncate_timezone_aware -from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids +from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids logger = logging.getLogger(__name__) diff --git a/dojo/finding/api/serializer.py b/dojo/finding/api/serializer.py index 49780338c86..c9f1b789703 100644 --- a/dojo/finding/api/serializer.py +++ b/dojo/finding/api/serializer.py @@ -323,7 +323,7 @@ def get_attribute(self, instance): return instance def to_representation(self, finding): - from dojo.vulnerability_id.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- flag seam + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- flag seam return [{"vulnerability_id": value} for value in finding_vulnerability_id_strings(finding)] def to_internal_value(self, data): diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index 882d1d720cc..969e686980a 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -9,7 +9,7 @@ from dojo.celery import app from dojo.models import Endpoint_Status, Finding, System_Settings -from dojo.vulnerability_id.queries import vulnerability_id_prefetch +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index 85586ba252a..c22fe37f70d 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -55,7 +55,7 @@ get_object_or_none, to_str_typed, ) -from dojo.vulnerability_id.manager import persist_for_finding +from dojo.vulnerability.manager import persist_for_finding logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 42721500a56..8f7b567d949 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -707,9 +707,13 @@ def copy(self, test=None): copy.tags.set(old_tags) # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util). # Route vulnerability ids through the dual-write seam so the copy gets legacy rows AND - # entity references (copy.cve was already carried over by copy_model_util). - from dojo.vulnerability_id.manager import persist_for_finding # noqa: PLC0415 -- avoid import cycle - vulnerability_id_strings = [row.vulnerability_id for row in self.vulnerability_id_set.all()] + # entity references (copy.cve was already carried over by copy_model_util). Read the SOURCE + # ids through the flag seam too (not the legacy relation directly), so copy() keeps working + # after the legacy Vulnerability_Id store is retired. + from dojo.vulnerability.manager import persist_for_finding # noqa: PLC0415 -- avoid import cycle + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 + + vulnerability_id_strings = finding_vulnerability_id_strings(self) if vulnerability_id_strings: persist_for_finding(copy, vulnerability_id_strings, delete_existing=False) for finding_cwe in self.finding_cwe_set.all(): @@ -842,7 +846,7 @@ def _get_unsaved_vulnerability_ids(finding) -> str: def _get_saved_vulnerability_ids(finding) -> str: if finding.id is not None: - from dojo.vulnerability_id.queries import use_entity_reads # noqa: PLC0415 + from dojo.vulnerability.queries import use_entity_reads # noqa: PLC0415 if use_entity_reads(): # Entity store: the prefetch-honoring reverse relation (vulnerability_references # + select_related("vulnerability")) so Prefetch is honored — no N+1 in dedupe. @@ -1393,7 +1397,7 @@ def get_references_with_links(self): @cached_property def vulnerability_ids(self): - from dojo.vulnerability_id.queries import use_entity_reads # noqa: PLC0415 + from dojo.vulnerability.queries import use_entity_reads # noqa: PLC0415 # Get vulnerability ids from database and convert to list of strings if use_entity_reads(): # Entity store: reverse relation is ordered by FindingVulnerabilityReference.Meta.ordering. diff --git a/dojo/finding/queries.py b/dojo/finding/queries.py index 2d849ad56e7..3ddc4ab477b 100644 --- a/dojo/finding/queries.py +++ b/dojo/finding/queries.py @@ -20,7 +20,7 @@ def get_auth_filter(key): return None Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task -from dojo.vulnerability_id.queries import vulnerability_id_prefetch +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) diff --git a/dojo/finding/ui/views.py b/dojo/finding/ui/views.py index a1f73dc2e5c..401766fb907 100644 --- a/dojo/finding/ui/views.py +++ b/dojo/finding/ui/views.py @@ -121,7 +121,7 @@ reopen_external_issue, update_external_issue, ) -from dojo.vulnerability_id.queries import vulnerability_id_prefetch +from dojo.vulnerability.queries import vulnerability_id_prefetch JFORM_PUSH_TO_JIRA_MESSAGE = "jform.push_to_jira: %s" diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index d2afbe8cd8c..db6ec0932eb 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -41,7 +41,7 @@ from dojo.tools.factory import get_parser from dojo.tools.parser_test import ParserTest from dojo.utils import max_safe -from dojo.vulnerability_id.manager import VulnerabilityIdManager +from dojo.vulnerability.manager import VulnerabilityIdManager logger = logging.getLogger(__name__) diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index f6ea16006ba..cffd8eb8e10 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -988,8 +988,13 @@ def reconcile_vulnerability_ids( # while vulnerability_ids do not, and vice versa). self.reconcile_cwes(finding) - # Use prefetched data directly without triggering queries - existing_vuln_ids = {v.vulnerability_id for v in finding.vulnerability_id_set.all()} + # Read the existing ids through the flag seam (entity references when the flag is on, legacy + # rows when off) instead of the legacy relation directly, so reconcile keeps working after + # the legacy Vulnerability_Id store is retired. The prefetch is seam-matched upstream (the + # reimport finding query uses vulnerability_id_prefetch()), so this stays a no-query read. + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- avoid import cycle + + existing_vuln_ids = set(finding_vulnerability_id_strings(finding)) new_vuln_ids = set(vulnerability_ids_to_process) # Early exit if unchanged — no DB work needed diff --git a/dojo/management/commands/dedupe.py b/dojo/management/commands/dedupe.py index f2d9468705a..36d46a1919b 100644 --- a/dojo/management/commands/dedupe.py +++ b/dojo/management/commands/dedupe.py @@ -20,7 +20,7 @@ get_system_setting, mass_model_updater, ) -from dojo.vulnerability_id.queries import vulnerability_id_prefetch +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") diff --git a/dojo/management/commands/migrate_vulnerability_ids.py b/dojo/management/commands/migrate_vulnerability_ids.py index 8027b52d4d1..6f64bfcc740 100644 --- a/dojo/management/commands/migrate_vulnerability_ids.py +++ b/dojo/management/commands/migrate_vulnerability_ids.py @@ -3,7 +3,7 @@ from django.core.management.base import BaseCommand from django.db import connection -from dojo.vulnerability_id.backfill import DEFAULT_WINDOW_SIZE, run_backfill +from dojo.vulnerability.backfill import DEFAULT_WINDOW_SIZE, run_backfill logger = logging.getLogger(__name__) @@ -11,12 +11,12 @@ class Command(BaseCommand): """ - Backfill the VulnerabilityId entity + FindingVulnerabilityReference tables from the legacy + Backfill the Vulnerability entity + FindingVulnerabilityReference tables from the legacy dojo_vulnerability_id table and dojo_finding.cve. Idempotent and resumable (insert-only + ON CONFLICT DO NOTHING), so it is safe to run repeatedly: large tenants can pre-run it on the previous release, then it is an instant - catch-up during the deploy-time migration. Shares dojo.vulnerability_id.backfill.run_backfill + catch-up during the deploy-time migration. Shares dojo.vulnerability.backfill.run_backfill with migration 0286. """ diff --git a/dojo/management/commands/verify_vulnerability_ids.py b/dojo/management/commands/verify_vulnerability_ids.py index 4c106efd0b7..e9b3b28516f 100644 --- a/dojo/management/commands/verify_vulnerability_ids.py +++ b/dojo/management/commands/verify_vulnerability_ids.py @@ -4,14 +4,14 @@ from django.core.management.base import BaseCommand from django.db.models import Q -from dojo.models import Finding, FindingVulnerabilityReference, Vulnerability_Id, VulnerabilityId +from dojo.models import Finding, FindingVulnerabilityReference, Vulnerability, Vulnerability_Id logger = logging.getLogger(__name__) # Both stores, with their roles — printed so operators know exactly what is being compared. -LEGACY_TABLE = "dojo_vulnerability_id" # legacy per-finding rows (authoritative for writes) -ENTITY_TABLE = "dojo_vulnerabilityid" # normalized VulnerabilityId registry -REFERENCE_TABLE = "dojo_findingvulnerabilityreference" # ordered Finding -> VulnerabilityId links +LEGACY_TABLE = "dojo_vulnerability_id" # legacy per-finding rows (authoritative for writes) +ENTITY_TABLE = "dojo_vulnerability" # normalized Vulnerability registry +REFERENCE_TABLE = "dojo_findingvulnerabilityreference" # ordered Finding -> Vulnerability links class Command(BaseCommand): @@ -22,8 +22,12 @@ class Command(BaseCommand): Checks per sampled finding: (1) every legacy (finding, id) pair has a matching reference; extra references are tolerated ONLY when they are cve-only-derived (string == finding.cve); - (2) the order-0 reference string equals finding.cve wherever cve is set. Plus row / entity / - reference / orphan-entity counts. + (2) the order-0 reference string equals finding.cve. Check (2) is a hard discrepancy ONLY when + finding.cve IS among the finding's reference ids but not at order 0 (a genuine ordering bug). + When finding.cve is NOT among the finding's ids (legacy anomaly: cve set to a value that was + never in the id list), order-0 == cve is unreachable WITHOUT breaking the hash invariant — the + reference set must equal the legacy id set for byte-parity, so cve cannot be injected — and it + is reported as a warning, not drift. Plus row / entity / reference / orphan-entity counts. """ help = "Verify the legacy vulnerability-id store matches the entity/reference store (nonzero exit on drift)." @@ -40,17 +44,17 @@ def add_arguments(self, parser): def handle(self, *args, **options): sample = options["sample"] discrepancies = [] + warnings = [] counts = { f"legacy_rows ({LEGACY_TABLE})": Vulnerability_Id.objects.count(), - f"entities ({ENTITY_TABLE})": VulnerabilityId.objects.count(), + f"entities ({ENTITY_TABLE})": Vulnerability.objects.count(), f"references ({REFERENCE_TABLE})": FindingVulnerabilityReference.objects.count(), - "orphan_entities": VulnerabilityId.objects.filter(finding_references__isnull=True).count(), + "orphan_entities": Vulnerability.objects.filter(finding_references__isnull=True).count(), } sampled = ( - Finding.objects - .filter( + Finding.objects.filter( # Reverse relation query names: legacy FK -> "vulnerability_id", entity refs -> # "vulnerability_references" (the "_set" form is the attribute accessor, not a lookup). Q(vulnerability_id__isnull=False) @@ -76,20 +80,35 @@ def handle(self, *args, **options): extras = reference_strings - legacy_strings unexpected = {value for value in extras if value != finding.cve} if unexpected: - discrepancies.append(f"finding {finding.id}: references with no legacy row (not cve-derived): {sorted(unexpected)}") + discrepancies.append( + f"finding {finding.id}: references with no legacy row (not cve-derived): {sorted(unexpected)}", + ) if finding.cve: primary = next((ref for ref in references if ref.order == 0), None) primary_string = primary.vulnerability.vulnerability_id if primary else None if primary_string != finding.cve: - discrepancies.append( - f"finding {finding.id}: order-0 reference {primary_string!r} != cve {finding.cve!r}", - ) + if finding.cve in reference_strings: + # cve IS one of the finding's ids but is not at order 0 — a genuine + # ordering bug the backfill/dual-write should have prevented. + discrepancies.append( + f"finding {finding.id}: order-0 reference {primary_string!r} != cve {finding.cve!r}", + ) + else: + # cve is not among the finding's vulnerability ids (legacy anomaly). Forcing + # order-0 == cve would require injecting cve into the reference set, which + # would diverge from the legacy id set and break the sacred hash invariant. + # Parity holds; the order-0 == cve invariant simply does not apply here. + warnings.append( + f"finding {finding.id}: cve {finding.cve!r} is not among its vulnerability ids " + f"(order-0 is {primary_string!r}); hash parity preserved, order-0==cve not applicable", + ) report = { "counts": counts, "findings_checked": checked, "discrepancies": discrepancies, + "warnings": warnings, "ok": not discrepancies, } @@ -106,6 +125,11 @@ def handle(self, *args, **options): self.stdout.write(self.style.ERROR(f" - {line}")) else: self.stdout.write(self.style.SUCCESS(" no discrepancies")) + if warnings: + # Non-fatal: reported for visibility but NOT counted toward the nonzero exit. + self.stdout.write(self.style.WARNING(f" {len(warnings)} warnings (non-fatal):")) + for line in warnings: + self.stdout.write(self.style.WARNING(f" - {line}")) if discrepancies: # Nonzero exit for CI / ring-flip gating. diff --git a/dojo/models.py b/dojo/models.py index bd3f4ac3386..5d338257160 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -407,9 +407,9 @@ class Meta: Finding_Template, Vulnerability_Id, # noqa: F401 -- re-export ) -from dojo.vulnerability_id.models import ( # noqa: E402 -- re-export; FKs reference dojo.Finding / dojo.VulnerabilityId by string +from dojo.vulnerability.models import ( # noqa: E402 -- re-export; FKs reference dojo.Finding / dojo.Vulnerability by string FindingVulnerabilityReference, # noqa: F401 -- re-export - VulnerabilityId, # noqa: F401 -- re-export + Vulnerability, # noqa: F401 -- re-export ) diff --git a/dojo/risk_acceptance/api/mixins.py b/dojo/risk_acceptance/api/mixins.py index e4c4dc6e0b2..bf8c8f4b0cc 100644 --- a/dojo/risk_acceptance/api/mixins.py +++ b/dojo/risk_acceptance/api/mixins.py @@ -14,7 +14,7 @@ from dojo.engagement.queries import get_authorized_engagements from dojo.models import Engagement, Risk_Acceptance, User from dojo.risk_acceptance.api.serializer import RiskAcceptanceSerializer -from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids +from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids AcceptedRisk = NamedTuple("AcceptedRisk", (("vulnerability_id", str), ("justification", str), ("accepted_by", str))) diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index f494cdd928d..cfe9dee3879 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -288,7 +288,7 @@ DD_REQUESTS_TIMEOUT=(int, 30), # Dictates if v3 functionality will be enabled (on by default as of 3.0.0; set to False to revert to the legacy Endpoint model) DD_V3_FEATURE_LOCATIONS=(bool, True), - # Dictates if findings read their vulnerability ids from the normalized VulnerabilityId entity/reference tables + # Dictates if findings read their vulnerability ids from the normalized Vulnerability entity/reference tables # (on by default; set to False to read from the legacy Vulnerability_Id table). Writes are always dual, so the flag # only selects the READ store and is reversible in both directions with zero data drift. DD_V3_FEATURE_VULNERABILITY_IDS=(bool, True), diff --git a/dojo/test/ui/views.py b/dojo/test/ui/views.py index 70d8d866fbf..a48738d2451 100644 --- a/dojo/test/ui/views.py +++ b/dojo/test/ui/views.py @@ -82,7 +82,7 @@ process_tag_notifications, redirect_to_return_url_or_else, ) -from dojo.vulnerability_id.queries import vulnerability_id_prefetch +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) parse_logger = logging.getLogger("dojo") diff --git a/dojo/vulnerability/__init__.py b/dojo/vulnerability/__init__.py new file mode 100644 index 00000000000..739e09bf6cf --- /dev/null +++ b/dojo/vulnerability/__init__.py @@ -0,0 +1 @@ +import dojo.vulnerability.admin # noqa: F401 diff --git a/dojo/vulnerability_id/admin.py b/dojo/vulnerability/admin.py similarity index 58% rename from dojo/vulnerability_id/admin.py rename to dojo/vulnerability/admin.py index 1f44929c958..a135c3ce4f4 100644 --- a/dojo/vulnerability_id/admin.py +++ b/dojo/vulnerability/admin.py @@ -1,12 +1,12 @@ from django.contrib import admin -from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability -@admin.register(VulnerabilityId) +@admin.register(Vulnerability) class VulnerabilityIdAdmin(admin.ModelAdmin): - """Admin support for the VulnerabilityId entity model.""" + """Admin support for the Vulnerability entity model.""" @admin.register(FindingVulnerabilityReference) diff --git a/dojo/vulnerability_id/backfill.py b/dojo/vulnerability/backfill.py similarity index 70% rename from dojo/vulnerability_id/backfill.py rename to dojo/vulnerability/backfill.py index e7b1ee1a00a..4d67c6c01d6 100644 --- a/dojo/vulnerability_id/backfill.py +++ b/dojo/vulnerability/backfill.py @@ -1,5 +1,5 @@ """ -Idempotent, set-based backfill of the VulnerabilityId entity + FindingVulnerabilityReference +Idempotent, set-based backfill of the Vulnerability entity + FindingVulnerabilityReference tables from the legacy dojo_vulnerability_id table and dojo_finding.cve. Shared by migration 0286 (RunPython forward) and the `migrate_vulnerability_ids` management @@ -11,6 +11,7 @@ The reference `order` is assigned cve-first then legacy-PK-ascending, so ``order == 0`` is the finding's cve (primary id) by construction — matching every live write path. """ + import logging from django.db import connection @@ -21,7 +22,7 @@ _TYPE_UPDATE_BATCH = 1000 _INSERT_ENTITIES_FROM_LEGACY = """ - INSERT INTO dojo_vulnerabilityid (vulnerability_id, vulnerability_id_type, created, updated) + INSERT INTO dojo_vulnerability (vulnerability_id, vulnerability_id_type, created, updated) SELECT vulnerability_id, MAX(vulnerability_id_type), now(), now() FROM dojo_vulnerability_id GROUP BY vulnerability_id @@ -29,7 +30,7 @@ """ _INSERT_ENTITIES_FROM_CVE = """ - INSERT INTO dojo_vulnerabilityid (vulnerability_id, vulnerability_id_type, created, updated) + INSERT INTO dojo_vulnerability (vulnerability_id, vulnerability_id_type, created, updated) SELECT DISTINCT cve, NULL, now(), now() FROM dojo_finding WHERE cve IS NOT NULL AND cve != '' @@ -37,7 +38,7 @@ """ # Windowed by legacy finding_id. ROW_NUMBER orders cve-match first, then legacy PK ascending; -# s.ord - 1 makes order 0-based so the cve row lands at order 0. The join to dojo_vulnerabilityid +# s.ord - 1 makes order 0-based so the cve row lands at order 0. The join to dojo_vulnerability # resolves each legacy string to its entity PK. ON CONFLICT covers re-runs and the unique # (finding_id, vulnerability_id) constraint (dojo 0282 guarantees legacy pair uniqueness, so no # dedupe layer is needed here). @@ -54,21 +55,18 @@ JOIN dojo_finding f ON f.id = v.finding_id WHERE v.finding_id >= %(lo)s AND v.finding_id < %(hi)s ) s - JOIN dojo_vulnerabilityid e ON e.vulnerability_id = s.vulnerability_id + JOIN dojo_vulnerability e ON e.vulnerability_id = s.vulnerability_id ON CONFLICT (finding_id, vulnerability_id) DO NOTHING """ -# Findings that carry a cve but have NO legacy vulnerability_id rows (so the windowed pass above -# created no reference for them): give each an order-0 reference to its cve entity. -_INSERT_CVE_ONLY_REFERENCES = """ - INSERT INTO dojo_findingvulnerabilityreference (finding_id, vulnerability_id, "order", created) - SELECT f.id, e.id, 0, now() - FROM dojo_finding f - JOIN dojo_vulnerabilityid e ON e.vulnerability_id = f.cve - LEFT JOIN dojo_findingvulnerabilityreference r ON r.finding_id = f.id - WHERE f.cve IS NOT NULL AND f.cve != '' AND r.id IS NULL - ON CONFLICT (finding_id, vulnerability_id) DO NOTHING -""" +# NOTE: cve-only findings (cve set, but NO legacy vulnerability_id rows) deliberately get NO +# reference. The reference set MUST mirror the legacy per-finding row set exactly, or the hash input +# (Finding.get_vulnerability_ids(), a sorted join of the reference strings) diverges from the legacy +# read and the finding's dedup identity silently changes when the read flag flips. The live +# dual-write never fabricates a bare-cve reference either, so backfill must not. Their cve still +# lives in the entity registry (step 2, as an orphan entity) and the cve field is untouched; only +# the reference is withheld. verify_vulnerability_ids treats these as a non-fatal warning +# (order-0 == cve is N/A when cve is not among the finding's ids). def _stamp_missing_types(cursor, logger): @@ -81,19 +79,17 @@ def _stamp_missing_types(cursor, logger): pass idempotent (a re-run produces zero UPDATEs). """ cursor.execute( - "SELECT id, vulnerability_id FROM dojo_vulnerabilityid WHERE vulnerability_id_type IS NULL", + "SELECT id, vulnerability_id FROM dojo_vulnerability WHERE vulnerability_id_type IS NULL", ) rows = cursor.fetchall() updates = [ - (resolved, pk) - for pk, vuln_id in rows - if (resolved := resolve_vulnerability_id_type(vuln_id)) is not None + (resolved, pk) for pk, vuln_id in rows if (resolved := resolve_vulnerability_id_type(vuln_id)) is not None ] stamped = 0 for i in range(0, len(updates), _TYPE_UPDATE_BATCH): - batch = updates[i:i + _TYPE_UPDATE_BATCH] + batch = updates[i : i + _TYPE_UPDATE_BATCH] cursor.executemany( - "UPDATE dojo_vulnerabilityid SET vulnerability_id_type = %s WHERE id = %s", + "UPDATE dojo_vulnerability SET vulnerability_id_type = %s WHERE id = %s", batch, ) stamped += len(batch) @@ -111,16 +107,16 @@ def run_backfill(*, logger=None, window_size=DEFAULT_WINDOW_SIZE): counts = {} with connection.cursor() as cursor: - logger.info("Backfill step 1/4: entities from legacy dojo_vulnerability_id") + logger.info("Backfill step 1/3: entities from legacy dojo_vulnerability_id") cursor.execute(_INSERT_ENTITIES_FROM_LEGACY) counts["entities_from_legacy"] = cursor.rowcount - logger.info("Backfill step 2/4: cve-only entities from dojo_finding.cve") + logger.info("Backfill step 2/3: cve entities from dojo_finding.cve (registry; may be orphan)") cursor.execute(_INSERT_ENTITIES_FROM_CVE) counts["entities_from_cve"] = cursor.rowcount counts["types_stamped"] = _stamp_missing_types(cursor, logger) - logger.info("Backfill step 3/4: references (windowed by finding_id, window=%s)", window_size) + logger.info("Backfill step 3/3: references (windowed by finding_id, window=%s)", window_size) cursor.execute("SELECT MIN(finding_id), MAX(finding_id) FROM dojo_vulnerability_id") lo, hi_max = cursor.fetchone() references = 0 @@ -132,9 +128,5 @@ def run_backfill(*, logger=None, window_size=DEFAULT_WINDOW_SIZE): lo = hi counts["references_from_legacy"] = references - logger.info("Backfill step 4/4: order-0 references for cve-only findings") - cursor.execute(_INSERT_CVE_ONLY_REFERENCES) - counts["references_from_cve_only"] = cursor.rowcount - logger.info("Backfill complete: %s", counts) return counts diff --git a/dojo/vulnerability_id/manager.py b/dojo/vulnerability/manager.py similarity index 85% rename from dojo/vulnerability_id/manager.py rename to dojo/vulnerability/manager.py index c7e7c1256bd..85f7845b5e0 100644 --- a/dojo/vulnerability_id/manager.py +++ b/dojo/vulnerability/manager.py @@ -2,21 +2,22 @@ Write seam for vulnerability ids: an UNCONDITIONAL dual-write. Every writer persists the legacy ``Vulnerability_Id`` rows byte-for-byte as today AND the new -``VulnerabilityId`` entity + ``FindingVulnerabilityReference`` rows, through this one module. The -``V3_FEATURE_VULNERABILITY_IDS`` flag gates READS only (see dojo.vulnerability_id.queries), so the +``Vulnerability`` entity + ``FindingVulnerabilityReference`` rows, through this one module. The +``V3_FEATURE_VULNERABILITY_IDS`` flag gates READS only (see dojo.vulnerability.queries), so the two stores never drift and the flag is reversible in both directions. The legacy-write blocks are marked ``# TODO: Delete after Vulnerability_Id retirement``. CWE buffering (store_cwes/pending_cwes/reconcile_cwes) is deliberately NOT owned here — it keeps riding the importer flush boundary exactly as before. """ + from collections.abc import Iterable from django.db import transaction from dojo.finding.models import Vulnerability_Id from dojo.finding.vulnerability_id import resolve_vulnerability_id_type -from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability def _clean(vulnerability_ids: Iterable[str]) -> list[str]: @@ -26,7 +27,7 @@ def _clean(vulnerability_ids: Iterable[str]) -> list[str]: def bulk_get_or_create_entities(strings: Iterable[str]) -> dict[str, int]: """ - Map each id string to its ``VulnerabilityId`` PK, creating any that are missing. + Map each id string to its ``Vulnerability`` PK, creating any that are missing. Race-safe by design (a deliberate departure from the locations rollback-on-race pattern): shared global id strings collide routinely across concurrent imports, so we @@ -37,13 +38,13 @@ def bulk_get_or_create_entities(strings: Iterable[str]) -> dict[str, int]: if not unique: return {} mapping = dict( - VulnerabilityId.objects.filter(vulnerability_id__in=unique).values_list("vulnerability_id", "id"), + Vulnerability.objects.filter(vulnerability_id__in=unique).values_list("vulnerability_id", "id"), ) missing = [s for s in unique if s not in mapping] if missing: - VulnerabilityId.objects.bulk_create( + Vulnerability.objects.bulk_create( [ - VulnerabilityId(vulnerability_id=s, vulnerability_id_type=resolve_vulnerability_id_type(s)) + Vulnerability(vulnerability_id=s, vulnerability_id_type=resolve_vulnerability_id_type(s)) for s in missing ], ignore_conflicts=True, @@ -52,7 +53,7 @@ def bulk_get_or_create_entities(strings: Iterable[str]) -> dict[str, int]: # bulk_create(ignore_conflicts=True) does not return usable PKs on Postgres — refetch. mapping.update( dict( - VulnerabilityId.objects.filter(vulnerability_id__in=missing).values_list("vulnerability_id", "id"), + Vulnerability.objects.filter(vulnerability_id__in=missing).values_list("vulnerability_id", "id"), ), ) return mapping @@ -85,7 +86,9 @@ def persist_for_finding(finding, vulnerability_ids, *, delete_existing: bool = T Vulnerability_Id.objects.filter(finding=finding).delete() Vulnerability_Id.objects.bulk_create( [ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) + Vulnerability_Id( + finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid), + ) for vid in ids ], batch_size=1000, @@ -129,7 +132,9 @@ def flush(self) -> None: if self.pending_deletes: Vulnerability_Id.objects.filter(finding_id__in=self.pending_deletes).delete() legacy_rows = [ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) + Vulnerability_Id( + finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid), + ) for finding, ids in self.pending for vid in ids ] diff --git a/dojo/vulnerability/models.py b/dojo/vulnerability/models.py new file mode 100644 index 00000000000..aa00f45ae8f --- /dev/null +++ b/dojo/vulnerability/models.py @@ -0,0 +1,86 @@ +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector +from django.core.validators import MaxValueValidator, MinValueValidator +from django.db import models +from django.db.models.functions import Upper + + +class Vulnerability(models.Model): + + """ + Global registry of vulnerability identifier strings (CVE-..., GHSA-..., ...). + + Vocabulary (Dependency-Track/OSV/Snyk convention): a **Finding** is the occurrence in your + environment; a **Vulnerability** is the definition it references. This entity is that + definition, whose natural key happens to be the identifier string ``vulnerability_id``. + + Naming trap: the string column is kept named ``vulnerability_id`` (wire-frozen), while + ``FindingVulnerabilityReference.vulnerability`` is an FK to this model whose attname/column is + ALSO ``vulnerability_id``. Legacy code that filtered the reverse relation with *strings* via + ``vulnerability_id__in=`` must now go through ``vulnerability__vulnerability_id__in``; + a bare ``vulnerability_id__in`` on the reference now means "FK pk in". + + Stores each identifier EXACTLY as supplied (never case-normalized): the string is + simultaneously the natural key, the display form, and the hash_code input, so any + normalization would change Finding.get_vulnerability_ids() output and churn hashes. + Case-insensitive matching is an index concern (see Upper index), not storage. + Rows are never deleted by finding write paths — orphan entities are the (future) + enrichment registry. EPSS/KEV columns are NULL until enrichment populates them + (NULL = "not enriched", distinct from False). + """ + + vulnerability_id = models.TextField(unique=True, blank=False, null=False, verbose_name="Vulnerability Id") + # Autodetected prefix (CVE, GHSA, ...) via dojo.finding.vulnerability_id. + # resolve_vulnerability_id_type; stamped at entity creation. Excluded from hashing. + vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, editable=False, db_index=True) + epss_score = models.FloatField( + null=True, blank=True, default=None, validators=[MinValueValidator(0.0), MaxValueValidator(1.0)], + ) + epss_percentile = models.FloatField( + null=True, blank=True, default=None, validators=[MinValueValidator(0.0), MaxValueValidator(1.0)], + ) + known_exploited = models.BooleanField(null=True, blank=True, default=None) + ransomware_used = models.BooleanField(null=True, blank=True, default=None) + kev_date = models.DateField(null=True, blank=True, default=None) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + GinIndex(SearchVector("vulnerability_id", weight="A", config="english"), name="dojo_vulnid_entity_fts_gin"), + GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vulnid_entity_trgm"), + models.Index(Upper("vulnerability_id"), name="dojo_vulnid_entity_upper"), + ] + + def __str__(self): + return self.vulnerability_id + + +class FindingVulnerabilityReference(models.Model): + + """Ordered link Finding -> Vulnerability. order == 0 is the primary id (== Finding.cve).""" + + finding = models.ForeignKey( + "dojo.Finding", on_delete=models.CASCADE, related_name="vulnerability_references", editable=False, + ) + vulnerability = models.ForeignKey( + "dojo.Vulnerability", on_delete=models.CASCADE, related_name="finding_references", editable=False, + ) + order = models.PositiveSmallIntegerField( + default=0, help_text="Position in the finding's id list; 0 is the primary identifier.", + ) + created = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["order"] + constraints = [ + models.UniqueConstraint(fields=["finding", "vulnerability"], name="unique_finding_vulnerability"), + # Structural guarantee of a single primary id per finding: no two references may share + # an order (in particular, never two order-0 rows). Compatible with the delete-then- + # insert dual-write, which assigns 0..n-1 in one transaction (see manager.py). + models.UniqueConstraint(fields=["finding", "order"], name="unique_finding_order"), + ] + indexes = [models.Index(fields=["finding", "order"]), models.Index(fields=["vulnerability"])] + + def __str__(self): + return self.vulnerability.vulnerability_id diff --git a/dojo/vulnerability_id/queries.py b/dojo/vulnerability/queries.py similarity index 77% rename from dojo/vulnerability_id/queries.py rename to dojo/vulnerability/queries.py index 0b31b07aeca..4a2cfe5d764 100644 --- a/dojo/vulnerability_id/queries.py +++ b/dojo/vulnerability/queries.py @@ -6,10 +6,13 @@ try: from dojo.authorization.query_filters import get_auth_filter except ImportError: - def get_auth_filter(key): return None + + def get_auth_filter(key): + return None + from dojo.finding.models import Vulnerability_Id -from dojo.vulnerability_id.models import FindingVulnerabilityReference, VulnerabilityId +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability logger = logging.getLogger(__name__) @@ -34,15 +37,11 @@ def finding_ids_with_vulnerability_ids(values, *, lookup="in"): Backs both filter functions in dojo/filters.py and the risk-acceptance exact-match lookup. """ if use_entity_reads(): - return ( - FindingVulnerabilityReference.objects - .filter(**{f"vulnerability__vulnerability_id__{lookup}": values}) - .values_list("finding_id", flat=True) - ) - return ( - Vulnerability_Id.objects - .filter(**{f"vulnerability_id__{lookup}": values}) - .values_list("finding_id", flat=True) + return FindingVulnerabilityReference.objects.filter( + **{f"vulnerability__vulnerability_id__{lookup}": values}, + ).values_list("finding_id", flat=True) + return Vulnerability_Id.objects.filter(**{f"vulnerability_id__{lookup}": values}).values_list( + "finding_id", flat=True, ) @@ -80,24 +79,17 @@ def first_vulnerability_id_subquery(): the lowest-PK legacy row. """ if use_entity_reads(): - return ( - FindingVulnerabilityReference.objects - .filter(finding=OuterRef("pk"), order=0) - .values("vulnerability__vulnerability_id")[:1] - ) - return ( - Vulnerability_Id.objects - .filter(finding=OuterRef("pk")) - .order_by("id") - .values("vulnerability_id")[:1] - ) + return FindingVulnerabilityReference.objects.filter(finding=OuterRef("pk"), order=0).values( + "vulnerability__vulnerability_id", + )[:1] + return Vulnerability_Id.objects.filter(finding=OuterRef("pk")).order_by("id").values("vulnerability_id")[:1] def get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): impl = get_auth_filter("vulnerability_id.get_authorized_entities") if impl: return impl(permission, queryset=queryset, user=user) - return VulnerabilityId.objects.all().order_by("id") if queryset is None else queryset + return Vulnerability.objects.all().order_by("id") if queryset is None else queryset def get_authorized_finding_vulnerability_references(permission, queryset=None, user=None): diff --git a/dojo/vulnerability_id/__init__.py b/dojo/vulnerability_id/__init__.py deleted file mode 100644 index 27909fcd3bf..00000000000 --- a/dojo/vulnerability_id/__init__.py +++ /dev/null @@ -1 +0,0 @@ -import dojo.vulnerability_id.admin # noqa: F401 diff --git a/dojo/vulnerability_id/models.py b/dojo/vulnerability_id/models.py deleted file mode 100644 index 84e58b28baa..00000000000 --- a/dojo/vulnerability_id/models.py +++ /dev/null @@ -1,71 +0,0 @@ -from django.contrib.postgres.indexes import GinIndex -from django.contrib.postgres.search import SearchVector -from django.core.validators import MaxValueValidator, MinValueValidator -from django.db import models -from django.db.models.functions import Upper - - -class VulnerabilityId(models.Model): - - """ - Global registry of vulnerability identifier strings (CVE-..., GHSA-..., ...). - - Stores each identifier EXACTLY as supplied (never case-normalized): the string is - simultaneously the natural key, the display form, and the hash_code input, so any - normalization would change Finding.get_vulnerability_ids() output and churn hashes. - Case-insensitive matching is an index concern (see Upper index), not storage. - Rows are never deleted by finding write paths — orphan entities are the (future) - enrichment registry. EPSS/KEV columns are NULL until enrichment populates them - (NULL = "not enriched", distinct from False). - """ - - vulnerability_id = models.TextField(unique=True, blank=False, null=False, - verbose_name="Vulnerability Id") - # Autodetected prefix (CVE, GHSA, ...) via dojo.finding.vulnerability_id. - # resolve_vulnerability_id_type; stamped at entity creation. Excluded from hashing. - vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, - editable=False, db_index=True) - epss_score = models.FloatField(null=True, blank=True, default=None, - validators=[MinValueValidator(0.0), MaxValueValidator(1.0)]) - epss_percentile = models.FloatField(null=True, blank=True, default=None, - validators=[MinValueValidator(0.0), MaxValueValidator(1.0)]) - known_exploited = models.BooleanField(null=True, blank=True, default=None) - ransomware_used = models.BooleanField(null=True, blank=True, default=None) - kev_date = models.DateField(null=True, blank=True, default=None) - created = models.DateTimeField(auto_now_add=True) - updated = models.DateTimeField(auto_now=True) - - class Meta: - indexes = [ - GinIndex(SearchVector("vulnerability_id", weight="A", config="english"), - name="dojo_vulnid_entity_fts_gin"), - GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], - name="dojo_vulnid_entity_trgm"), - models.Index(Upper("vulnerability_id"), name="dojo_vulnid_entity_upper"), - ] - - def __str__(self): - return self.vulnerability_id - - -class FindingVulnerabilityReference(models.Model): - - """Ordered link Finding -> VulnerabilityId. order == 0 is the primary id (== Finding.cve).""" - - finding = models.ForeignKey("dojo.Finding", on_delete=models.CASCADE, - related_name="vulnerability_references", editable=False) - vulnerability = models.ForeignKey("dojo.VulnerabilityId", on_delete=models.CASCADE, - related_name="finding_references", editable=False) - order = models.PositiveSmallIntegerField(default=0, - help_text="Position in the finding's id list; 0 is the primary identifier.") - created = models.DateTimeField(auto_now_add=True) - - class Meta: - ordering = ["order"] - constraints = [models.UniqueConstraint(fields=["finding", "vulnerability"], - name="unique_finding_vulnerability")] - indexes = [models.Index(fields=["finding", "order"]), - models.Index(fields=["vulnerability"])] - - def __str__(self): - return self.vulnerability.vulnerability_id diff --git a/unittests/test_migrations.py b/unittests/test_migrations.py index d476a4bac97..e6a0481891d 100644 --- a/unittests/test_migrations.py +++ b/unittests/test_migrations.py @@ -16,10 +16,10 @@ Product_Type, Test, Test_Type, + Vulnerability, Vulnerability_Id, - VulnerabilityId, ) -from dojo.vulnerability_id.backfill import run_backfill +from dojo.vulnerability.backfill import run_backfill @skip("Outdated - this class was testing some version of migration; it is not needed anymore") @@ -54,8 +54,8 @@ def prepare(self): self.finding = Finding.objects.create(test_id=self.test.pk, reporter_id=user).pk self.endpoint = Endpoint.objects.create(host="foo.bar", product_id=self.product.pk).pk self.endpoint_status = Endpoint_Status.objects.create( - finding_id=self.finding, - endpoint_id=self.endpoint, + finding_id=self.finding, + endpoint_id=self.endpoint, ).pk Endpoint.objects.get(id=self.endpoint).endpoint_status.add( Endpoint_Status.objects.get(id=self.endpoint_status), @@ -182,11 +182,12 @@ def test_after_migration(self): class TestVulnerabilityIdBackfill(TestCase): """ - dojo.vulnerability_id.backfill.run_backfill (the routine migration 0286 delegates to) turns - legacy dojo_vulnerability_id rows + dojo_finding.cve into VulnerabilityId entities and ordered + dojo.vulnerability.backfill.run_backfill (the routine migration 0286 delegates to) turns + legacy dojo_vulnerability_id rows + dojo_finding.cve into Vulnerability entities and ordered FindingVulnerabilityReference rows (order 0 == cve). Covers ordered ids, cve-drift (cve is not - the lowest-PK row), case-variant ids (2 distinct entities), cve-only findings (no legacy rows), - null-cve findings, and an idempotent re-run. + the lowest-PK row), case-variant ids (2 distinct entities), cve-only findings (no legacy rows => + NO reference, so the reference set mirrors the legacy row set and the hash stays byte-identical; + the cve remains an orphan registry entity), null-cve findings, and an idempotent re-run. A plain TestCase rather than MigratorTestCase: django_test_migrations replays the full squashed migration chain from a wiped schema in setUp, which is unusable in this codebase (the replay @@ -247,7 +248,8 @@ def add_legacy(finding, vulnerability_id): for vid in ("CVE-C1", "cve-c1"): add_legacy(cls.finding_c, vid) - # D: cve-only — cve set, NO legacy rows => step-4 order-0 ref to the cve entity. + # D: cve-only — cve set, NO legacy rows => NO reference (mirror the empty legacy row set for + # hash parity); CVE-D1 still becomes an orphan registry entity via step 2. cls.finding_d = make_finding("D", "CVE-D1") # E: legacy rows but NULL cve => order by PK, no cve-only ref. @@ -265,21 +267,27 @@ def test_backfill(self): run_backfill() expected_entities = { - "CVE-A1", "CVE-A2", "CVE-A3", - "CVE-B1", "CVE-B2", "CVE-B3", - "CVE-C1", "cve-c1", + "CVE-A1", + "CVE-A2", + "CVE-A3", + "CVE-B1", + "CVE-B2", + "CVE-B3", + "CVE-C1", + "cve-c1", "CVE-D1", - "CVE-E1", "CVE-E2", + "CVE-E1", + "CVE-E2", } with self.subTest("entity extraction (exact-cased, case-variant = 2 entities)"): self.assertEqual( - set(VulnerabilityId.objects.values_list("vulnerability_id", flat=True)), + set(Vulnerability.objects.values_list("vulnerability_id", flat=True)), expected_entities, ) with self.subTest("entity type carried / stamped from prefix"): - for entity in VulnerabilityId.objects.all(): + for entity in Vulnerability.objects.all(): self.assertEqual( entity.vulnerability_id_type, resolve_vulnerability_id_type(entity.vulnerability_id), @@ -294,26 +302,30 @@ def test_backfill(self): with self.subTest("C: case-variant ids both referenced, exact cve at order 0"): self.assertEqual(self._order_map(self.finding_c), {0: "CVE-C1", 1: "cve-c1"}) - with self.subTest("D: cve-only finding gets an order-0 reference"): - self.assertEqual(self._order_map(self.finding_d), {0: "CVE-D1"}) + with self.subTest("D: cve-only finding gets NO reference (parity: mirrors empty legacy set)"): + self.assertEqual(self._order_map(self.finding_d), {}) + # The cve is still catalogued as an orphan registry entity (never referenced). + cve_d = Vulnerability.objects.get(vulnerability_id="CVE-D1") + self.assertFalse(cve_d.finding_references.exists()) with self.subTest("E: null-cve finding ordered by PK, no cve-only ref"): self.assertEqual(self._order_map(self.finding_e), {0: "CVE-E1", 1: "CVE-E2"}) - with self.subTest("pair sets: every legacy pair is in the new store; extras are cve-only"): + with self.subTest("pair sets: the reference set EXACTLY mirrors the legacy row set"): legacy_pairs = set(Vulnerability_Id.objects.values_list("finding_id", "vulnerability_id")) new_pairs = { (ref.finding_id, ref.vulnerability.vulnerability_id) for ref in FindingVulnerabilityReference.objects.all() } - self.assertTrue(legacy_pairs.issubset(new_pairs)) - self.assertEqual(new_pairs - legacy_pairs, {(self.finding_d.id, "CVE-D1")}) + # No fabricated (cve-only) references: the two stores hold identical (finding, id) pairs, + # which is exactly what keeps get_vulnerability_ids() byte-identical across the flag. + self.assertEqual(new_pairs, legacy_pairs) with self.subTest("idempotency: re-running the backfill changes nothing"): - entities_before = VulnerabilityId.objects.count() + entities_before = Vulnerability.objects.count() refs_before = FindingVulnerabilityReference.objects.count() counts = run_backfill() - self.assertEqual(VulnerabilityId.objects.count(), entities_before) + self.assertEqual(Vulnerability.objects.count(), entities_before) self.assertEqual(FindingVulnerabilityReference.objects.count(), refs_before) self.assertEqual( counts, @@ -322,6 +334,5 @@ def test_backfill(self): "entities_from_cve": 0, "types_stamped": 0, "references_from_legacy": 0, - "references_from_cve_only": 0, }, ) diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 9594e1dfeec..dcf50c51f2c 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -598,7 +598,7 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # store + bulk flush) and +10 reimport-with-new (per-finding reconcile reads # existing Finding_CWE rows for each changed finding). # Vulnerability_Id entity dual-write (unconditional): +2 import / +12 - # reimport-no-change / +6 reimport-with-new queries (VulnerabilityId entity + + # reimport-no-change / +6 reimport-with-new queries (Vulnerability entity + # FindingVulnerabilityReference bulk writes alongside the legacy rows; batched, # not per-finding). EXPECTED_ZAP_IMPORT_V2 = 296 diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index bb892df08ac..cc3e09716dd 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -3,12 +3,17 @@ Fixture-free (no dojo_testdata.json) so they run in both the OSS CI DB and the Pro container. They assert the NEW behavior that the legacy Vulnerability_Id tests do not: that persist_for_finding -and VulnerabilityIdManager write the VulnerabilityId entity + ordered FindingVulnerabilityReference +and VulnerabilityIdManager write the Vulnerability entity + ordered FindingVulnerabilityReference rows alongside the legacy rows, keep order 0 == first id, reconcile both stores, and never delete entity rows (orphans are retained). """ + +import pathlib +from io import StringIO + from django.contrib.auth import get_user_model from django.core.management import call_command +from django.db import IntegrityError, transaction from django.test import TestCase, override_settings from django.utils.timezone import now @@ -23,11 +28,12 @@ Product_Type, Test, Test_Type, + Vulnerability, Vulnerability_Id, - VulnerabilityId, ) -from dojo.vulnerability_id.manager import VulnerabilityIdManager, persist_for_finding -from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids +from dojo.tools.factory import get_parser +from dojo.vulnerability.manager import VulnerabilityIdManager, persist_for_finding +from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids class VulnerabilityIdDualWriteTestCase(TestCase): @@ -40,20 +46,30 @@ def setUpTestData(cls): # description / scan_type / environment are required under full model validation # (base save full_clean when V3_FEATURE_LOCATIONS is on — the CI "(true)" matrix leg). cls.product = Product.objects.create( - prod_type=cls.prod_type, name="vulnid-dw-prod", description="vuln-id dual-write tests", + prod_type=cls.prod_type, + name="vulnid-dw-prod", + description="vuln-id dual-write tests", ) cls.engagement = Engagement.objects.create(product=cls.product, target_start=now(), target_end=now()) cls.test_type = Test_Type.objects.create(name="vulnid-dw-tt") cls.environment = Development_Environment.objects.get_or_create(name="Development")[0] cls.test = Test.objects.create( - engagement=cls.engagement, test_type=cls.test_type, scan_type="vulnid-dw-tt", - environment=cls.environment, target_start=now(), target_end=now(), + engagement=cls.engagement, + test_type=cls.test_type, + scan_type="vulnid-dw-tt", + environment=cls.environment, + target_start=now(), + target_end=now(), ) cls.user = get_user_model().objects.create(username="vulnid-dw-user") def _make_finding(self, title): return Finding.objects.create( - test=self.test, reporter=self.user, title=title, severity="High", numerical_severity="S0", + test=self.test, + reporter=self.user, + title=title, + severity="High", + numerical_severity="S0", ) def _legacy(self, finding): @@ -67,7 +83,6 @@ def _refs(self, finding): class TestPersistForFinding(VulnerabilityIdDualWriteTestCase): - def test_dual_write_and_reconcile(self): finding = self._make_finding("persist") @@ -76,8 +91,8 @@ def test_dual_write_and_reconcile(self): with self.subTest("legacy rows written (deduped)"): self.assertEqual(self._legacy(finding), ["CVE-2021-1", "CVE-2021-2"]) with self.subTest("entity rows created"): - self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-2021-1").exists()) - self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-2021-2").exists()) + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-1").exists()) + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-2").exists()) with self.subTest("references ordered, order 0 == first id"): self.assertEqual(self._refs(finding), {0: "CVE-2021-1", 1: "CVE-2021-2"}) @@ -89,7 +104,7 @@ def test_dual_write_and_reconcile(self): with self.subTest("references reconciled with new order"): self.assertEqual(self._refs(finding), {0: "CVE-2021-2", 1: "CVE-2021-9"}) with self.subTest("dropped id's entity is retained (orphan registry)"): - self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-2021-1").exists()) + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-1").exists()) def test_empty_clears_both_stores(self): finding = self._make_finding("persist-empty") @@ -102,7 +117,6 @@ def test_empty_clears_both_stores(self): class TestSaveVulnerabilityIdsIntegration(VulnerabilityIdDualWriteTestCase): - def test_save_vulnerability_ids_dual_writes_and_syncs_cve(self): finding = self._make_finding("helper") save_vulnerability_ids(finding, ["CVE-2023-1", "CVE-2023-2"], delete_existing=False) @@ -113,7 +127,6 @@ def test_save_vulnerability_ids_dual_writes_and_syncs_cve(self): class TestVulnerabilityIdManager(VulnerabilityIdDualWriteTestCase): - def test_batch_record_and_reconcile_flush(self): finding_new = self._make_finding("mgr-new") finding_reimport = self._make_finding("mgr-reimport") @@ -148,7 +161,11 @@ class TestReadSeamFlagParity(VulnerabilityIdDualWriteTestCase): def setUpTestData(cls): super().setUpTestData() cls.finding = Finding.objects.create( - test=cls.test, reporter=cls.user, title="parity", severity="High", numerical_severity="S0", + test=cls.test, + reporter=cls.user, + title="parity", + severity="High", + numerical_severity="S0", ) save_vulnerability_ids(cls.finding, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) cls.finding.save() @@ -218,7 +235,7 @@ def test_migrate_command_backfills_idempotently(self): # cve-match lands at order 0. self.assertEqual(self._refs(finding), {0: "CVE-CMD-1", 1: "CVE-CMD-2"}) - self.assertTrue(VulnerabilityId.objects.filter(vulnerability_id="CVE-CMD-1").exists()) + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-CMD-1").exists()) # Idempotent re-run: no change. call_command("migrate_vulnerability_ids") @@ -241,6 +258,40 @@ def test_verify_command_fails_on_discrepancy(self): with self.assertRaises(SystemExit): call_command("verify_vulnerability_ids") + def test_verify_warns_not_fails_when_cve_absent_from_ids(self): + # Anomalous pre-upgrade data: the finding's cve is set to a value that is NOT among its + # legacy vulnerability-id rows. The backfill cannot make order-0 == cve without injecting + # cve into the reference set, which would diverge from the legacy id set and break the hash + # invariant. verify must therefore treat order-0 != cve as a NON-FATAL warning here (not a + # hard discrepancy), while parity between the two stores still holds. + finding = self._make_finding("cmd-verify-cve-absent") + finding.cve = "CVE-ABSENT-9" + finding.save() + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-PRESENT-1") + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-PRESENT-2") + + call_command("migrate_vulnerability_ids") + + # order-0 is a legacy row, NOT the cve (which is not among the ids at all). + refs = self._refs(finding) + self.assertNotEqual(refs[0], finding.cve) + self.assertNotIn(finding.cve, refs.values()) + + # verify does NOT raise (warning path), and reports the finding as a non-fatal warning. + out = StringIO() + call_command("verify_vulnerability_ids", stdout=out) + output = out.getvalue() + self.assertIn("no discrepancies", output) + self.assertIn("warnings (non-fatal)", output) + self.assertIn("CVE-ABSENT-9", output) + + # Hash parity across the flag still holds (the reference set == the legacy id set). + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off_hash = Finding.objects.get(pk=finding.pk).get_vulnerability_ids() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on_hash = Finding.objects.get(pk=finding.pk).get_vulnerability_ids() + self.assertEqual(off_hash, on_hash) + class TestUpgradeReadParity(VulnerabilityIdDualWriteTestCase): @@ -323,5 +374,374 @@ def test_backfilled_cve_not_first_row_preserves_hash_and_property(self): self.assertEqual(off_prop, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # cve first, then PK-asc rest # Raw v2 wire: same set, different order (the documented, bounded divergence). self.assertEqual(off_wire, ["CVE-M-1", "CVE-M-2", "CVE-M-3"]) # legacy PK order - self.assertEqual(on_wire, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # entity cve-first order + self.assertEqual(on_wire, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # entity cve-first order self.assertEqual(set(off_wire), set(on_wire)) + + +class TestReferenceOrderUniqueness(VulnerabilityIdDualWriteTestCase): + + """ + The unique(finding, order) constraint structurally guarantees a single primary id per + finding — in particular, no two order-0 references for the same finding. + """ + + def test_two_order_zero_references_rejected(self): + finding = self._make_finding("order-unique") + entity_a = Vulnerability.objects.create(vulnerability_id="CVE-ORD-A") + entity_b = Vulnerability.objects.create(vulnerability_id="CVE-ORD-B") + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_a, order=0) + # A second reference at order 0 for the same finding violates unique(finding, order). + with self.assertRaises(IntegrityError), transaction.atomic(): + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_b, order=0) + + def test_distinct_orders_allowed(self): + finding = self._make_finding("order-distinct") + entity_a = Vulnerability.objects.create(vulnerability_id="CVE-ORD-C") + entity_b = Vulnerability.objects.create(vulnerability_id="CVE-ORD-D") + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_a, order=0) + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_b, order=1) + self.assertEqual(self._refs(finding), {0: "CVE-ORD-C", 1: "CVE-ORD-D"}) + + +class TestLegacyStoreIndependence(VulnerabilityIdDualWriteTestCase): + + """ + CUTOVER SAFETY PROOF: entity reads never touch the legacy Vulnerability_Id store. + + This is THE property that makes retiring the dual-write (a one-way conversion) safe. With the + read flag ON, deleting a finding's legacy rows while leaving its entity references intact must + not change ANY read surface. If poisoning legacy changes nothing, nothing reads it, so the + legacy write can be dropped without touching behavior. Runs flag-ON only, because that is the + post-cutover read path. All read surfaces are checked in one pass (subTest) so every dependency + on legacy surfaces at once rather than one failure at a time. + """ + + def _surfaces(self, fid): + # Fresh instance per read: vulnerability_ids is a cached_property. + return { + "get_vulnerability_ids (hash input)": Finding.objects.get(pk=fid).get_vulnerability_ids(), + "compute_hash_code": Finding.objects.get(pk=fid).compute_hash_code(), + "vulnerability_ids property": sorted(Finding.objects.get(pk=fid).vulnerability_ids), + "v2 wire": sorted( + r["vulnerability_id"] for r in VulnerabilityIdsField().to_representation(Finding.objects.get(pk=fid)) + ), + "filter __in finds finding": fid + in set(finding_ids_with_vulnerability_ids(["CVE-IND-1", "CVE-IND-2", "CVE-IND-3"])), + "filter __exact finds finding": fid in set(finding_ids_with_vulnerability_ids("CVE-IND-2", lookup="exact")), + } + + @override_settings(V3_FEATURE_VULNERABILITY_IDS=True) + def test_reads_are_independent_of_legacy_store(self): + finding = self._make_finding("poison") + save_vulnerability_ids(finding, ["CVE-IND-1", "CVE-IND-2", "CVE-IND-3"]) + finding.save() + fid = finding.pk + + control = self._surfaces(fid) + # Sanity: the finding is genuinely discoverable through the seam before poisoning. + self.assertEqual(control["get_vulnerability_ids (hash input)"], "CVE-IND-1CVE-IND-2CVE-IND-3") + self.assertTrue(control["filter __in finds finding"]) + + # Poison: delete ALL legacy rows for this finding; entity references stay intact. + deleted, _ = Vulnerability_Id.objects.filter(finding_id=fid).delete() + self.assertEqual(deleted, 3) + self.assertEqual(Vulnerability_Id.objects.filter(finding_id=fid).count(), 0) + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding_id=fid).count(), 3) + + after = self._surfaces(fid) + for surface, value in control.items(): + with self.subTest(surface=surface): + self.assertEqual( + after[surface], + value, + f"read surface {surface!r} changed after poisoning the legacy store -> it still reads legacy", + ) + + @override_settings(V3_FEATURE_VULNERABILITY_IDS=True) + def test_full_lifecycle_with_zero_legacy_rows(self): + # Model the post-cutover world directly: write through the seam, immediately drop the legacy + # rows (so only entity refs exist, as if the legacy write had been removed), and exercise the + # lifecycle. Reads must be correct with NO legacy rows ever backing them. + finding = self._make_finding("entity-only") + save_vulnerability_ids(finding, ["CVE-EO-1", "CVE-EO-2"]) + finding.save() + Vulnerability_Id.objects.filter(finding=finding).delete() + + f = Finding.objects.get(pk=finding.pk) + self.assertEqual(f.get_vulnerability_ids(), "CVE-EO-1CVE-EO-2") + self.assertEqual(sorted(f.vulnerability_ids), ["CVE-EO-1", "CVE-EO-2"]) + + # Reconcile (reimport-style) still works with no legacy backing. + save_vulnerability_ids(f, ["CVE-EO-2", "CVE-EO-9"], delete_existing=True) + f.save() + Vulnerability_Id.objects.filter(finding=finding).delete() + self.assertEqual(Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), "CVE-EO-2CVE-EO-9") + + +class TestDedupDifferential(VulnerabilityIdDualWriteTestCase): + + """ + CUTOVER SAFETY PROOF: switching the read store cannot move a deduplication decision. + + Dedup keys on hash_code, whose ONLY vulnerability-id-dependent term is get_vulnerability_ids() + (a sorted join of the id set -- order-independent by construction). So proving that function is + byte-identical flag-off vs flag-on across a battery of id shapes (including the anomalies), plus + proving the realized compute_hash_code() is flag-identical, means dedup identity is invariant + under the cutover. Assertions target get_vulnerability_ids() rather than compute_hash_code for + the shape/order checks so they stay meaningful regardless of per-scanner hash configuration. + """ + + SHAPES = [ + ("single", ["CVE-DD-1"]), + ("multi ascending", ["CVE-DD-1", "CVE-DD-2", "CVE-DD-3"]), + ("multi unsorted", ["CVE-DD-3", "CVE-DD-1", "CVE-DD-2"]), + ("mixed prefixes", ["GHSA-dd-zzzz", "CVE-DD-9", "PYSEC-2021-1"]), + ("case variants (distinct, preserved)", ["CVE-DD-1", "cve-dd-1"]), + ("duplicates collapse", ["CVE-DD-7", "CVE-DD-7", "CVE-DD-8"]), + ("large set", [f"CVE-DDBIG-{i:04d}" for i in range(60)]), + ] + + def _hash_input(self, fid, *, entity): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=entity): + return Finding.objects.get(pk=fid).get_vulnerability_ids() + + def test_hash_input_identical_across_flag_and_shapes(self): + for label, ids in self.SHAPES: + with self.subTest(shape=label): + finding = self._make_finding(f"dd-{label}") + save_vulnerability_ids(finding, ids) + finding.save() + legacy_read = self._hash_input(finding.pk, entity=False) + entity_read = self._hash_input(finding.pk, entity=True) + self.assertEqual( + legacy_read, + entity_read, + f"get_vulnerability_ids diverged across the read store for shape {label!r} " + f"-> dedup identity would move", + ) + # The hash input is the sorted join, so it is exactly the canonical dedup key. + self.assertEqual(entity_read, "".join(sorted(dict.fromkeys(ids)))) + + def test_hash_input_is_order_independent_under_entity_reads(self): + f1 = self._make_finding("dd-order-1") + f2 = self._make_finding("dd-order-2") + save_vulnerability_ids(f1, ["CVE-ORDER-1", "CVE-ORDER-2", "CVE-ORDER-3"]) + save_vulnerability_ids(f2, ["CVE-ORDER-3", "CVE-ORDER-1", "CVE-ORDER-2"]) + f1.save() + f2.save() + self.assertEqual( + self._hash_input(f1.pk, entity=True), + self._hash_input(f2.pk, entity=True), + "same id-set in different order produced different hash inputs under entity reads", + ) + + def test_realized_hash_code_is_flag_identical(self): + # The realized compute_hash_code() (whatever fields the environment hashes -- OSS reads + # settings.HASHCODE_FIELDS_PER_SCANNER, Pro reads its DB-configured dedupe fields) must be + # invariant when the read store flips, because its ONLY vulnerability-id-dependent input is + # get_vulnerability_ids(), proven byte-identical across the flag above. This asserts that + # invariance directly on the real hash the deduper keys on, in whatever the running config is. + finding = self._make_finding("dd-hashcode") + finding.cwe = 79 # non-zero so a null-cwe config never forces the legacy fallback + save_vulnerability_ids(finding, ["CVE-HC-1", "CVE-HC-2"]) + finding.save() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off = Finding.objects.get(pk=finding.pk).compute_hash_code() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on = Finding.objects.get(pk=finding.pk).compute_hash_code() + self.assertEqual(off, on, "compute_hash_code diverged across the read store -> dedup decision could move") + + def test_reconcile_tracks_id_change_identically(self): + finding = self._make_finding("dd-reconcile") + save_vulnerability_ids(finding, ["CVE-RC-1", "CVE-RC-2"]) + finding.save() + self.assertEqual(self._hash_input(finding.pk, entity=False), self._hash_input(finding.pk, entity=True)) + + # Reimport-style change of the id set. + save_vulnerability_ids(Finding.objects.get(pk=finding.pk), ["CVE-RC-2", "CVE-RC-9"], delete_existing=True) + self.assertEqual(self._hash_input(finding.pk, entity=False), self._hash_input(finding.pk, entity=True)) + self.assertEqual(self._hash_input(finding.pk, entity=True), "CVE-RC-2CVE-RC-9") + + +class TestRealScannerCorpus(VulnerabilityIdDualWriteTestCase): + + """ + CUTOVER SAFETY PROOF against REAL scanner output, not hand-written shapes. + + Parses shipped sample reports from real scanners through their real parsers and drives every + finding's ACTUAL vulnerability-id list through the read seam. This is the closest offline + stand-in for a production bake: it exercises the id-string shapes scanners genuinely emit + (CVE-*, GHSA-*, Debian TEMP-*, Aqua AVD-KSV-*, no-prefix ids, intra-report duplicates, large + volumes) rather than the shapes a test author thought to enumerate. For every real finding it + asserts (1) get_vulnerability_ids() is byte-identical flag-off vs flag-on, and (2) it is + independent of the legacy store (poisoning legacy changes nothing). Fixture-free: uses only the + parsers + the sample files that ship in unittests/scans/. + """ + + # (scan_type, path-under-unittests/scans) verified to yield real vulnerability ids. + CORPUS = [ + ("Trivy Scan", "trivy/legacy_many_vulns.json"), # CVE-* + Debian TEMP-* + ("Anchore Grype", "anchore_grype/many_vulns2.json"), # high-volume CVE-* + ("Anchore Engine Scan", "anchore_engine/new_format_issue_11552.json"), # modern CVE-* + ("OSV Scan", "osv_scanner/many_findings.json"), # GHSA-* + ("Snyk Scan", "snyk/single_project_upgrade_libs.json"), # CVE-* + ("NPM Audit Scan", "npm_audit/multiple_cwes.json"), # CVE-* + ("Aqua Scan", "aqua/aqua_devops_issue_10611.json"), # CVE-* + ("Trivy Operator Scan", "trivy_operator/cis_benchmark.json"), # Aqua AVD-KSV-* (no CVE) + ] + + # Bound per report so the suite stays fast while still spanning every source. + PER_REPORT = 12 + + def _parse_real_id_lists(self): + """Yield each real finding's actual vulnerability-id list from the sample corpus.""" + scans_dir = pathlib.Path(__file__).parent / "scans" + for scan_type, relpath in self.CORPUS: + path = scans_dir / relpath + if not path.is_file(): + continue + try: + parser = get_parser(scan_type) + with path.open("rb") as handle: + parsed = parser.get_findings(handle, None) + except Exception: # noqa: S112 -- a parser/sample drift must skip, not mask the real assertions + continue + emitted = 0 + for parsed_finding in parsed: + ids = [str(v) for v in (getattr(parsed_finding, "unsaved_vulnerability_ids", None) or [])] + if not ids: + continue + yield scan_type, ids + emitted += 1 + if emitted >= self.PER_REPORT: + break + + def test_real_corpus_read_parity_and_legacy_independence(self): + findings_checked = 0 + distinct_ids = set() + prefixes = set() + + for scan_type, real_ids in self._parse_real_id_lists(): + finding = self._make_finding(f"corpus-{scan_type}-{findings_checked}") + save_vulnerability_ids(finding, real_ids) + finding.save() + fid = finding.pk + + with self.subTest(scan_type=scan_type, sample_ids=real_ids[:3]): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + legacy_read = Finding.objects.get(pk=fid).get_vulnerability_ids() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + entity_read = Finding.objects.get(pk=fid).get_vulnerability_ids() + self.assertEqual(legacy_read, entity_read, "read store diverged on real scanner data") + + # Legacy-independence on real data: poison legacy, entity read must not budge. + Vulnerability_Id.objects.filter(finding_id=fid).delete() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + self.assertEqual( + Finding.objects.get(pk=fid).get_vulnerability_ids(), + entity_read, + "entity read changed after poisoning legacy on real scanner data", + ) + + findings_checked += 1 + distinct_ids.update(real_ids) + prefixes.update(vid.split("-")[0].upper() for vid in real_ids if "-" in vid) + + # Guard against a vacuous run: we must have exercised a real, diverse corpus. + self.assertGreaterEqual(findings_checked, 20, "corpus did not exercise enough real findings") + self.assertGreaterEqual(len(distinct_ids), 50, "corpus did not exercise enough distinct real ids") + self.assertGreaterEqual( + len({"CVE", "GHSA", "AVD"} & prefixes), + 2, + f"corpus lacked shape diversity; saw prefixes {sorted(prefixes)}", + ) + + +class TestEnumerableShapeParity(VulnerabilityIdDualWriteTestCase): + + """ + CUTOVER SAFETY: the enumerable edge shapes, each proven read-store parity + legacy-independent. + + Complements the real-scanner corpus with the deterministic anomalies a corpus may not contain: + odd id strings, and finding shapes where cve and the id set disagree. Every case asserts the + hash input (get_vulnerability_ids) is byte-identical flag-off vs flag-on, which is the property + the one-way cutover rests on. + """ + + STRING_ODDITIES = [ + ("whitespace-padded (stored verbatim)", [" CVE-WS-1 ", "CVE-WS-2"]), + ("unicode", ["CVE-Ünïcödé-1", "CVE-日本語-2"]), + ("punctuation / quotes", ["CVE-2024-o'brien", "CVE-%wild_", 'GHSA-a"b-cccc']), + ("case-only-differing (distinct)", ["CVE-CASE-1", "cve-case-1"]), + ("no prefix / bare number", ["12345", "not-a-cve", "GHSA-aaaa-bbbb-cccc"]), + # 50 chars: the legacy Vulnerability_Id.vulnerability_id CharField(max_length=50) ceiling, + # so this is the longest id the legacy store can physically hold (the entity is TextField). + ("max-length id (legacy 50-char ceiling)", ["CVE-2024-" + "9" * 41]), + ] + + def _parity(self, fid): + with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): + off = Finding.objects.get(pk=fid).get_vulnerability_ids() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + on = Finding.objects.get(pk=fid).get_vulnerability_ids() + return off, on + + def test_string_oddities_read_parity_and_independence(self): + for label, ids in self.STRING_ODDITIES: + with self.subTest(shape=label): + finding = self._make_finding(f"odd-{label}") + save_vulnerability_ids(finding, ids) + finding.save() + fid = finding.pk + + off, on = self._parity(fid) + self.assertEqual(off, on, f"read store diverged on shape {label!r}") + + # Poison legacy: entity read must not budge. + Vulnerability_Id.objects.filter(finding_id=fid).delete() + with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): + self.assertEqual(Finding.objects.get(pk=fid).get_vulnerability_ids(), on) + + def test_cve_only_finding_backfill_preserves_parity(self): + # Regression for the cve-only parity bug: a finding with cve set but NO legacy rows must get + # NO reference after backfill, so the entity read ("") matches the legacy read (""). The cve + # is still catalogued as an orphan registry entity. + finding = self._make_finding("shape-cve-only") + finding.cve = "CVE-SHAPE-ONLY" + finding.save() + self.assertEqual(Vulnerability_Id.objects.filter(finding=finding).count(), 0) + + call_command("migrate_vulnerability_ids") + + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) + self.assertTrue( + Vulnerability.objects.filter(vulnerability_id="CVE-SHAPE-ONLY", finding_references__isnull=True).exists(), + "cve should be catalogued as an orphan registry entity", + ) + off, on = self._parity(finding.pk) + self.assertEqual(off, on) + self.assertEqual(on, "", "cve-only finding must have an empty hash input in both stores") + + def test_null_cve_with_ids_backfill_parity(self): + finding = self._make_finding("shape-null-cve") + finding.cve = None + finding.save() + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-NULL-1") + Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-NULL-2") + + call_command("migrate_vulnerability_ids") + + off, on = self._parity(finding.pk) + self.assertEqual(off, on) + self.assertEqual(on, "CVE-NULL-1CVE-NULL-2") + + def test_copy_preserves_vulnerability_ids_across_flag(self): + src = self._make_finding("copy-src") + save_vulnerability_ids(src, ["CVE-CP-1", "CVE-CP-2"]) + src.save() + + duplicate = src.copy() + + off, on = self._parity(duplicate.pk) + self.assertEqual(off, on, "copied finding's read store diverged") + self.assertEqual(on, "CVE-CP-1CVE-CP-2") From 568154ed271205483572f2a9d6705e599f3e0842 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:08:34 -0600 Subject: [PATCH 11/18] Vulnerability id entity-only cutover: drop legacy Vulnerability_Id Read vulnerability ids exclusively from the Vulnerability entity + FindingVulnerabilityReference through-table. Remove the read flag and all flag-off branches, delete the legacy Vulnerability_Id model, and add migration 0287 dropping dojo_vulnerability_id (0286 backfills the entities first). - MF2: backfill no longer inserts cve-only references (preserves hash parity) - MF4: UniqueConstraint(finding, order) on the reference table - Reimporter reads finding_vulnerability_id_strings (fixes a latent N+1) - Repoint tests to the entity store; remove legacy-model coverage; delete the migrate_cve / verify_vulnerability_ids commands NOTE: test_tag_inheritance_perf EXPECTED_* query counts need recalibration under watson-enabled CI after the dual-write removal. Co-Authored-By: Claude Opus 4.8 (1M context) --- dojo/api_v2/prefetch/registrations.py | 3 - dojo/apps.py | 7 +- dojo/authorization/query_registrations.py | 15 - .../0287_delete_vulnerability_id.py | 20 + dojo/finding/admin.py | 7 - dojo/finding/api/serializer.py | 6 +- dojo/finding/models.py | 63 +-- dojo/finding/queries.py | 17 - dojo/importers/base_importer.py | 10 +- dojo/management/commands/migrate_cve.py | 56 -- .../commands/verify_vulnerability_ids.py | 137 ----- dojo/models.py | 1 - dojo/search/views.py | 6 +- dojo/settings/settings.dist.py | 5 - dojo/vulnerability/backfill.py | 13 +- dojo/vulnerability/manager.py | 48 +- dojo/vulnerability/queries.py | 61 +-- unittests/test_apply_finding_template.py | 4 +- unittests/test_authorization_queries.py | 36 +- .../test_authorization_queryset_coverage.py | 10 +- unittests/test_importers_importer.py | 33 +- unittests/test_importers_performance.py | 4 +- unittests/test_migrations.py | 175 ------ unittests/test_tag_inheritance_perf.py | 10 +- unittests/test_vulnerability_id.py | 505 +++--------------- unittests/test_vulnerability_id_type.py | 44 +- unittests/test_watson_index_prefetch.py | 6 - 27 files changed, 237 insertions(+), 1065 deletions(-) create mode 100644 dojo/db_migrations/0287_delete_vulnerability_id.py delete mode 100644 dojo/management/commands/migrate_cve.py delete mode 100644 dojo/management/commands/verify_vulnerability_ids.py diff --git a/dojo/api_v2/prefetch/registrations.py b/dojo/api_v2/prefetch/registrations.py index 8dc4faad7fb..7d2b7f58a3e 100644 --- a/dojo/api_v2/prefetch/registrations.py +++ b/dojo/api_v2/prefetch/registrations.py @@ -37,7 +37,6 @@ from dojo.engagement.queries import get_authorized_engagements from dojo.finding.queries import ( get_authorized_findings, - get_authorized_vulnerability_ids, ) from dojo.finding_group.queries import get_authorized_finding_groups from dojo.github.models import GITHUB_Issue, GITHUB_PKey @@ -94,7 +93,6 @@ Tool_Product_Settings, Tool_Type, UserContactInfo, - Vulnerability_Id, ) from dojo.notifications.models import Notification_Webhooks, Notifications from dojo.product.queries import ( @@ -223,7 +221,6 @@ # Models where we can simply fall back to a `get_authorized_*` method to check auth for model, helper in ( (Finding_Group, get_authorized_finding_groups), - (Vulnerability_Id, get_authorized_vulnerability_ids), (Vulnerability, get_authorized_vulnerability_id_entities), (FindingVulnerabilityReference, get_authorized_finding_vulnerability_references), ): diff --git a/dojo/apps.py b/dojo/apps.py index 3ea65c281f5..e50b3094375 100644 --- a/dojo/apps.py +++ b/dojo/apps.py @@ -113,11 +113,8 @@ def register_watson_models(app_config): watson.register(app_config.get_model("Location")) watson.register(app_config.get_model("Engagement"), fields=get_model_fields_with_extra(app_config.get_model("Engagement"), ("id", "product__name")), store=("product__name", )) watson.register(app_config.get_model("App_Analysis")) - # Watson stays on the legacy Vulnerability_Id table (unconditionally dual-written), so global - # search keeps working identically in both flag states with no buildwatson reindex. Moving the - # watson index (and the classic search view) to FindingVulnerabilityReference is a legacy-drop - # phase concern, not this reads-only flag flip. - watson.register(app_config.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", )) + # The legacy Vulnerability_Id table was retired (entity-only cutover); the classic watson-backed + # vuln-id search is gone. The Vue global search covers vulnerability ids via the entity store. # YourModel = app_config.get_model("YourModel") # watson.register(YourModel) diff --git a/dojo/authorization/query_registrations.py b/dojo/authorization/query_registrations.py index 420cabdc60c..36ec68e404c 100644 --- a/dojo/authorization/query_registrations.py +++ b/dojo/authorization/query_registrations.py @@ -38,7 +38,6 @@ Test, Test_Import, Tool_Product_Settings, - Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability @@ -405,20 +404,6 @@ def _get_authorized_findings(permission, queryset=None, user=None): register_auth_filter("finding.get_authorized_findings_for_queryset", _get_authorized_findings) -def _get_authorized_vulnerability_ids(permission, queryset=None, user=None): - user = _resolve_user(user) - qs = queryset if queryset is not None else Vulnerability_Id.objects.all() - if user is None or getattr(user, "is_anonymous", False): - return qs.none() - if _is_unrestricted(user, permission_to_action(permission)): - return qs - return qs.filter(finding__test__engagement__product__id__in=_authorized_product_ids(user)) - - -register_auth_filter("finding.get_authorized_vulnerability_ids", _get_authorized_vulnerability_ids) -register_auth_filter("finding.get_authorized_vulnerability_ids_for_queryset", _get_authorized_vulnerability_ids) - - def _get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): user = _resolve_user(user) qs = queryset if queryset is not None else Vulnerability.objects.all() diff --git a/dojo/db_migrations/0287_delete_vulnerability_id.py b/dojo/db_migrations/0287_delete_vulnerability_id.py new file mode 100644 index 00000000000..c0209fab36a --- /dev/null +++ b/dojo/db_migrations/0287_delete_vulnerability_id.py @@ -0,0 +1,20 @@ +# Entity-only cutover: drop the legacy Vulnerability_Id store. Vulnerability ids now live entirely in +# the Vulnerability entity + FindingVulnerabilityReference through-table (created in 0285, backfilled +# from dojo_vulnerability_id in 0286 — which runs BEFORE this drop). ⚠️ DESTRUCTIVE: drops +# dojo_vulnerability_id and its data. On upgrading installs, ensure 0286's backfill (or +# `manage.py migrate_vulnerability_ids` pre-run) has completed before this migration removes the source. + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("dojo", "0286_backfill_vulnerability_id_entities"), + ] + + operations = [ + migrations.DeleteModel( + name="Vulnerability_Id", + ), + ] diff --git a/dojo/finding/admin.py b/dojo/finding/admin.py index f10732d25f7..9561d1d1a89 100644 --- a/dojo/finding/admin.py +++ b/dojo/finding/admin.py @@ -6,7 +6,6 @@ Finding, Finding_Group, Finding_Template, - Vulnerability_Id, ) @@ -26,12 +25,6 @@ class FindingTemplateAdmin(admin.ModelAdmin): """Admin support for the Finding_Template model.""" -@admin.register(Vulnerability_Id) -class VulnerabilityIdAdmin(admin.ModelAdmin): - - """Admin support for the Vulnerability_Id model.""" - - @admin.register(Finding_Group) class FindingGroupAdmin(admin.ModelAdmin): diff --git a/dojo/finding/api/serializer.py b/dojo/finding/api/serializer.py index c9f1b789703..5d4b19b33ab 100644 --- a/dojo/finding/api/serializer.py +++ b/dojo/finding/api/serializer.py @@ -43,7 +43,7 @@ Test, Test_Type, User, - Vulnerability_Id, + Vulnerability, ) from dojo.notifications.helper import async_create_notification from dojo.user.queries import get_authorized_users @@ -297,7 +297,9 @@ def get_jira(self, obj): class VulnerabilityIdSerializer(serializers.ModelSerializer): class Meta: - model = Vulnerability_Id + # Schema-only (OpenAPI) shape for VulnerabilityIdsField; points at the Vulnerability entity, + # which carries the same ``vulnerability_id`` string column. + model = Vulnerability fields = ["vulnerability_id"] diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 8f7b567d949..6469118d879 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -25,7 +25,6 @@ from dojo.base_models.base import BaseModel from dojo.finding.cwe import cwe_label, finding_cwe_labels -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type # get_current_date/tomorrow/copy_model_util are defined early in dojo.models, before the # re-export that loads this module — so this resolves despite the partial circular load, and @@ -846,18 +845,11 @@ def _get_unsaved_vulnerability_ids(finding) -> str: def _get_saved_vulnerability_ids(finding) -> str: if finding.id is not None: - from dojo.vulnerability.queries import use_entity_reads # noqa: PLC0415 - if use_entity_reads(): - # Entity store: the prefetch-honoring reverse relation (vulnerability_references - # + select_related("vulnerability")) so Prefetch is honored — no N+1 in dedupe. - vulnerability_id_str_list = [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] - else: - # Use the reverse relation (vulnerability_id_set) rather than a fresh - # Vulnerability_Id.objects.filter(...) so prefetch_related("vulnerability_id_set") - # is honored — avoids an N+1 (COUNT + SELECT per finding) during dedupe/hashcode. - vulnerability_id_str_list = [str(vulnerability_id) for vulnerability_id in finding.vulnerability_id_set.all()] + # Entity store: the prefetch-honoring reverse relation (vulnerability_references + # + select_related("vulnerability")) so Prefetch is honored — no N+1 in dedupe. + vulnerability_id_str_list = [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] deduplicationLogger.debug("get_vulnerability_ids after the finding was saved. Vulnerability references count: " + str(len(vulnerability_id_str_list))) - # sort vulnerability_ids strings (no dedupe — byte parity with the legacy store) + # sort vulnerability_ids strings (no dedupe — ordering is irrelevant to the hash) return "".join(sorted(vulnerability_id_str_list)) return "" @@ -1397,14 +1389,9 @@ def get_references_with_links(self): @cached_property def vulnerability_ids(self): - from dojo.vulnerability.queries import use_entity_reads # noqa: PLC0415 - # Get vulnerability ids from database and convert to list of strings - if use_entity_reads(): - # Entity store: reverse relation is ordered by FindingVulnerabilityReference.Meta.ordering. - vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in self.vulnerability_references.all()] - else: - vulnerability_ids_model = self.vulnerability_id_set.all() - vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model] + # Get vulnerability ids from database and convert to list of strings. + # Entity store: reverse relation is ordered by FindingVulnerabilityReference.Meta.ordering. + vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in self.vulnerability_references.all()] # Synchronize the cve field with the unsaved_vulnerability_ids # We do this to be as flexible as possible to handle the fields until @@ -1452,39 +1439,9 @@ def set_hash_code(self, dedupe_option): deduplicationLogger.debug("Hash_code computed for finding: %s: %s", finding_id, self.hash_code) -class Vulnerability_Id(models.Model): - finding = models.ForeignKey("dojo.Finding", editable=False, on_delete=models.CASCADE) - vulnerability_id = models.TextField(max_length=50, blank=False, null=False) - # Autodetected from the id prefix (CVE, GHSA, ...); NULL when there is no non-numeric - # prefix. Denormalized/indexed so type-scoped queries (e.g. GROUP BY type) stay cheap. - vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, editable=False, db_index=True) - - class Meta: - constraints = [ - models.UniqueConstraint(fields=["finding", "vulnerability_id"], name="unique_finding_vulnerability_id"), - ] - indexes = [ - # Leading on vulnerability_id (the unique constraint's index leads on finding), for the - # vulnerability-id Explorer's GROUP BY vulnerability_id / lookups by exact id. - models.Index(fields=["vulnerability_id"], name="dojo_vuln_id_lookup_idx"), - # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. - GinIndex( - SearchVector("vulnerability_id", weight="A", config="english"), - name="dojo_vulnerability_id_fts_gin", - ), - GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vuln_id_trgm"), - ] - - def __str__(self): - return self.vulnerability_id - - def save(self, *args, **kwargs): - # bulk_create paths set the type at construction; this covers save()/get_or_create. - self.vulnerability_id_type = resolve_vulnerability_id_type(self.vulnerability_id) - super().save(*args, **kwargs) - - def get_absolute_url(self): - return reverse("view_finding", args=[str(self.finding.id)]) +# The legacy Vulnerability_Id model was removed in the entity-only cutover; vulnerability ids now +# live in the Vulnerability entity + FindingVulnerabilityReference through-table (dojo/vulnerability/). +# The dojo_vulnerability_id table is dropped by migration 0287. class Finding_CWE(models.Model): diff --git a/dojo/finding/queries.py b/dojo/finding/queries.py index 3ddc4ab477b..ab35a3a514b 100644 --- a/dojo/finding/queries.py +++ b/dojo/finding/queries.py @@ -17,7 +17,6 @@ def get_auth_filter(key): return None Endpoint_Status, Finding, Test_Import_Finding_Action, - Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task from dojo.vulnerability.queries import vulnerability_id_prefetch @@ -41,22 +40,6 @@ def get_authorized_findings_for_queryset(permission, queryset, user=None): return Finding.objects.all().order_by("id") if queryset is None else queryset -# Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request_or_task -def get_authorized_vulnerability_ids(permission, user=None): - impl = get_auth_filter("finding.get_authorized_vulnerability_ids") - if impl: - return impl(permission, user=user) - return Vulnerability_Id.objects.all() - - -def get_authorized_vulnerability_ids_for_queryset(permission, queryset, user=None): - impl = get_auth_filter("finding.get_authorized_vulnerability_ids_for_queryset") - if impl: - return impl(permission, queryset, user=user) - return queryset - - def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=True): """ Unified prefetch function for findings across the application. diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index db6ec0932eb..6f3d7349fea 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -82,8 +82,8 @@ def __init__( and will raise a `NotImplemented` exception """ ImporterOptions.__init__(self, *args, **kwargs) - # Unconditional dual-write seam: buffers legacy Vulnerability_Id rows AND the new - # entity/reference rows, flushed together at the batch boundary. + # Write seam: buffers the Vulnerability entity + FindingVulnerabilityReference rows, + # flushed at the batch boundary. self.vulnerability_id_manager = VulnerabilityIdManager() self.pending_cwes: list[Finding_CWE] = [] self.pending_cwe_deletes: list[int] = [] @@ -891,7 +891,7 @@ def store_vulnerability_ids( finding: Finding, ) -> Finding: """ - Accumulate Vulnerability_Id objects for bulk insert at the batch boundary. + Accumulate a finding's vulnerability-id references for bulk insert at the batch boundary. Call flush_vulnerability_ids() to persist. """ self.sanitize_vulnerability_ids(finding) @@ -926,8 +926,8 @@ def reconcile_cwes(self, finding: Finding) -> None: self.pending_cwes.extend([Finding_CWE(finding=finding, cwe=cwe) for cwe in new_cwes]) def flush_vulnerability_ids(self) -> None: - """Flush the dual-write vulnerability-id buffers, then the Finding_CWE buffers, and clear.""" - # Legacy Vulnerability_Id rows + entity/reference rows, in one transaction. + """Flush the vulnerability-id buffers, then the Finding_CWE buffers, and clear.""" + # Vulnerability entity + FindingVulnerabilityReference rows, in one transaction. self.vulnerability_id_manager.flush() # CWE buffers ride the same flush boundary as before (not owned by the manager). if self.pending_cwe_deletes: diff --git a/dojo/management/commands/migrate_cve.py b/dojo/management/commands/migrate_cve.py deleted file mode 100644 index eaa3985c19c..00000000000 --- a/dojo/management/commands/migrate_cve.py +++ /dev/null @@ -1,56 +0,0 @@ -import logging - -from django.core.management.base import BaseCommand - -from dojo.finding.helper import save_vulnerability_ids_template -from dojo.models import ( - Finding, - Finding_Template, - Vulnerability_Id, -) -from dojo.utils import mass_model_updater - -logger = logging.getLogger(__name__) - - -def create_vulnerability_id(finding): - Vulnerability_Id.objects.get_or_create( - finding=finding, vulnerability_id=finding.cve, - ) - - -def create_vulnerability_id_template(finding_template): - # Use the new TextField-based approach - if finding_template.cve: - save_vulnerability_ids_template(finding_template, [finding_template.cve]) - - -class Command(BaseCommand): - - """This management command creates vulnerability ids for all findings / findings_templates with cve's.""" - - help = "Usage: manage.py migrate_cve" - - def handle(self, *args, **options): - - logger.info("Starting migration of cves for Findings") - findings = Finding.objects.filter(cve__isnull=False) - mass_model_updater( - Finding, - findings, - create_vulnerability_id, - fields=None, - page_size=100, - log_prefix="creating vulnerability ids: ", - ) - - logger.info("Starting migration of cves for Finding_Templates") - finding_templates = Finding_Template.objects.filter(cve__isnull=False) - mass_model_updater( - Finding_Template, - finding_templates, - create_vulnerability_id_template, - fields=None, - page_size=100, - log_prefix="creating vulnerability ids: ", - ) diff --git a/dojo/management/commands/verify_vulnerability_ids.py b/dojo/management/commands/verify_vulnerability_ids.py deleted file mode 100644 index e9b3b28516f..00000000000 --- a/dojo/management/commands/verify_vulnerability_ids.py +++ /dev/null @@ -1,137 +0,0 @@ -import json -import logging - -from django.core.management.base import BaseCommand -from django.db.models import Q - -from dojo.models import Finding, FindingVulnerabilityReference, Vulnerability, Vulnerability_Id - -logger = logging.getLogger(__name__) - -# Both stores, with their roles — printed so operators know exactly what is being compared. -LEGACY_TABLE = "dojo_vulnerability_id" # legacy per-finding rows (authoritative for writes) -ENTITY_TABLE = "dojo_vulnerability" # normalized Vulnerability registry -REFERENCE_TABLE = "dojo_findingvulnerabilityreference" # ordered Finding -> Vulnerability links - - -class Command(BaseCommand): - - """ - Read-only reconciliation of the legacy vulnerability-id store against the entity/reference - store. Exits nonzero on any discrepancy, so it can gate a ring flip. - - Checks per sampled finding: (1) every legacy (finding, id) pair has a matching reference; - extra references are tolerated ONLY when they are cve-only-derived (string == finding.cve); - (2) the order-0 reference string equals finding.cve. Check (2) is a hard discrepancy ONLY when - finding.cve IS among the finding's reference ids but not at order 0 (a genuine ordering bug). - When finding.cve is NOT among the finding's ids (legacy anomaly: cve set to a value that was - never in the id list), order-0 == cve is unreachable WITHOUT breaking the hash invariant — the - reference set must equal the legacy id set for byte-parity, so cve cannot be injected — and it - is reported as a warning, not drift. Plus row / entity / reference / orphan-entity counts. - """ - - help = "Verify the legacy vulnerability-id store matches the entity/reference store (nonzero exit on drift)." - - def add_arguments(self, parser): - parser.add_argument( - "--sample", - type=int, - default=1000, - help="Number of findings to spot-check for pair parity / order-0 (default 1000).", - ) - parser.add_argument("--json", action="store_true", dest="as_json", help="Emit the report as JSON.") - - def handle(self, *args, **options): - sample = options["sample"] - discrepancies = [] - warnings = [] - - counts = { - f"legacy_rows ({LEGACY_TABLE})": Vulnerability_Id.objects.count(), - f"entities ({ENTITY_TABLE})": Vulnerability.objects.count(), - f"references ({REFERENCE_TABLE})": FindingVulnerabilityReference.objects.count(), - "orphan_entities": Vulnerability.objects.filter(finding_references__isnull=True).count(), - } - - sampled = ( - Finding.objects.filter( - # Reverse relation query names: legacy FK -> "vulnerability_id", entity refs -> - # "vulnerability_references" (the "_set" form is the attribute accessor, not a lookup). - Q(vulnerability_id__isnull=False) - | Q(vulnerability_references__isnull=False) - | (Q(cve__isnull=False) & ~Q(cve="")), - ) - .distinct() - .order_by("id") - .prefetch_related("vulnerability_id_set", "vulnerability_references__vulnerability")[:sample] - ) - - checked = 0 - for finding in sampled: - checked += 1 - legacy_strings = {row.vulnerability_id for row in finding.vulnerability_id_set.all()} - references = list(finding.vulnerability_references.all()) - reference_strings = {ref.vulnerability.vulnerability_id for ref in references} - - missing = legacy_strings - reference_strings - if missing: - discrepancies.append(f"finding {finding.id}: legacy ids missing a reference: {sorted(missing)}") - - extras = reference_strings - legacy_strings - unexpected = {value for value in extras if value != finding.cve} - if unexpected: - discrepancies.append( - f"finding {finding.id}: references with no legacy row (not cve-derived): {sorted(unexpected)}", - ) - - if finding.cve: - primary = next((ref for ref in references if ref.order == 0), None) - primary_string = primary.vulnerability.vulnerability_id if primary else None - if primary_string != finding.cve: - if finding.cve in reference_strings: - # cve IS one of the finding's ids but is not at order 0 — a genuine - # ordering bug the backfill/dual-write should have prevented. - discrepancies.append( - f"finding {finding.id}: order-0 reference {primary_string!r} != cve {finding.cve!r}", - ) - else: - # cve is not among the finding's vulnerability ids (legacy anomaly). Forcing - # order-0 == cve would require injecting cve into the reference set, which - # would diverge from the legacy id set and break the sacred hash invariant. - # Parity holds; the order-0 == cve invariant simply does not apply here. - warnings.append( - f"finding {finding.id}: cve {finding.cve!r} is not among its vulnerability ids " - f"(order-0 is {primary_string!r}); hash parity preserved, order-0==cve not applicable", - ) - - report = { - "counts": counts, - "findings_checked": checked, - "discrepancies": discrepancies, - "warnings": warnings, - "ok": not discrepancies, - } - - if options["as_json"]: - self.stdout.write(json.dumps(report, indent=2)) - else: - self.stdout.write("Vulnerability ID store reconciliation") - for label, value in counts.items(): - self.stdout.write(f" {label}: {value}") - self.stdout.write(f" findings checked: {checked}") - if discrepancies: - self.stdout.write(self.style.ERROR(f" {len(discrepancies)} discrepancies:")) - for line in discrepancies: - self.stdout.write(self.style.ERROR(f" - {line}")) - else: - self.stdout.write(self.style.SUCCESS(" no discrepancies")) - if warnings: - # Non-fatal: reported for visibility but NOT counted toward the nonzero exit. - self.stdout.write(self.style.WARNING(f" {len(warnings)} warnings (non-fatal):")) - for line in warnings: - self.stdout.write(self.style.WARNING(f" - {line}")) - - if discrepancies: - # Nonzero exit for CI / ring-flip gating. - msg = f"{len(discrepancies)} vulnerability-id reconciliation discrepancies" - raise SystemExit(msg) diff --git a/dojo/models.py b/dojo/models.py index 5d338257160..42a001d78dd 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -405,7 +405,6 @@ class Meta: Finding_CWE, # noqa: F401 -- re-export Finding_Group, # noqa: F401 -- re-export Finding_Template, - Vulnerability_Id, # noqa: F401 -- re-export ) from dojo.vulnerability.models import ( # noqa: E402 -- re-export; FKs reference dojo.Finding / dojo.Vulnerability by string FindingVulnerabilityReference, # noqa: F401 -- re-export diff --git a/dojo/search/views.py b/dojo/search/views.py index bd4220ca147..66afb8b9a96 100644 --- a/dojo/search/views.py +++ b/dojo/search/views.py @@ -13,7 +13,7 @@ from dojo.endpoint.queries import get_authorized_endpoints from dojo.endpoint.ui.views import prefetch_for_endpoints from dojo.engagement.queries import get_authorized_engagements -from dojo.finding.queries import get_authorized_findings, get_authorized_vulnerability_ids, prefetch_for_findings +from dojo.finding.queries import get_authorized_findings, prefetch_for_findings from dojo.finding.ui.filters import FindingFilter, FindingFilterWithoutObjectLookups from dojo.forms import FindingBulkUpdateForm, SimpleSearchForm from dojo.location.queries import get_authorized_locations, prefetch_for_locations @@ -139,7 +139,9 @@ def simple_search(request): authorized_endpoints = get_authorized_endpoints("view") authorized_finding_templates = Finding_Template.objects.all() authorized_app_analysis = get_authorized_app_analysis("view") - authorized_vulnerability_ids = get_authorized_vulnerability_ids("view") + # Legacy watson-backed vuln-id search is retired (this view returns 410); the entity + # store has no classic-search reader. Vue global search covers vulnerability ids. + authorized_vulnerability_ids = [] # TODO: better get findings in their own query and match on id. that would allow filtering on additional fields such prod_id, etc. diff --git a/dojo/settings/settings.dist.py b/dojo/settings/settings.dist.py index cfe9dee3879..b590576a62e 100644 --- a/dojo/settings/settings.dist.py +++ b/dojo/settings/settings.dist.py @@ -288,10 +288,6 @@ DD_REQUESTS_TIMEOUT=(int, 30), # Dictates if v3 functionality will be enabled (on by default as of 3.0.0; set to False to revert to the legacy Endpoint model) DD_V3_FEATURE_LOCATIONS=(bool, True), - # Dictates if findings read their vulnerability ids from the normalized Vulnerability entity/reference tables - # (on by default; set to False to read from the legacy Vulnerability_Id table). Writes are always dual, so the flag - # only selects the READ store and is reversible in both directions with zero data drift. - DD_V3_FEATURE_VULNERABILITY_IDS=(bool, True), # Dictates if v3 org/asset relabeling (+url routing) will be enabled (on by default as of 3.0.0; set to False to restore Product/Product Type labels and URLs) DD_ENABLE_V3_ORGANIZATION_ASSET_RELABEL=(bool, True), # Shared cache backend (django.core.cache). When set, Django uses RedisCache @@ -683,7 +679,6 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param # V3 Feature Flags V3_FEATURE_LOCATIONS = env("DD_V3_FEATURE_LOCATIONS") -V3_FEATURE_VULNERABILITY_IDS = env("DD_V3_FEATURE_VULNERABILITY_IDS") # ------------------------------------------------------------------------------ diff --git a/dojo/vulnerability/backfill.py b/dojo/vulnerability/backfill.py index 4d67c6c01d6..be4cc0d3f4c 100644 --- a/dojo/vulnerability/backfill.py +++ b/dojo/vulnerability/backfill.py @@ -60,13 +60,12 @@ """ # NOTE: cve-only findings (cve set, but NO legacy vulnerability_id rows) deliberately get NO -# reference. The reference set MUST mirror the legacy per-finding row set exactly, or the hash input -# (Finding.get_vulnerability_ids(), a sorted join of the reference strings) diverges from the legacy -# read and the finding's dedup identity silently changes when the read flag flips. The live -# dual-write never fabricates a bare-cve reference either, so backfill must not. Their cve still -# lives in the entity registry (step 2, as an orphan entity) and the cve field is untouched; only -# the reference is withheld. verify_vulnerability_ids treats these as a non-fatal warning -# (order-0 == cve is N/A when cve is not among the finding's ids). +# reference. The reference set MUST mirror the legacy per-finding row set exactly, or the finding's +# dedup identity (Finding.get_vulnerability_ids(), a sorted join of the reference strings) would +# change on upgrade vs. what the legacy reads produced. The live write path never fabricates a +# bare-cve reference either, so the backfill must not. Their cve still lives in the entity registry +# (step 2, as an orphan entity) and the cve field is untouched; only the reference is withheld +# (order-0 == cve is simply N/A when cve is not among the finding's ids). def _stamp_missing_types(cursor, logger): diff --git a/dojo/vulnerability/manager.py b/dojo/vulnerability/manager.py index 85f7845b5e0..e8bb628d17e 100644 --- a/dojo/vulnerability/manager.py +++ b/dojo/vulnerability/manager.py @@ -1,12 +1,9 @@ """ -Write seam for vulnerability ids: an UNCONDITIONAL dual-write. +Write seam for vulnerability ids: the ``Vulnerability`` entity + ``FindingVulnerabilityReference`` +rows are the single source of truth. Every writer persists them through this one module. -Every writer persists the legacy ``Vulnerability_Id`` rows byte-for-byte as today AND the new -``Vulnerability`` entity + ``FindingVulnerabilityReference`` rows, through this one module. The -``V3_FEATURE_VULNERABILITY_IDS`` flag gates READS only (see dojo.vulnerability.queries), so the -two stores never drift and the flag is reversible in both directions. - -The legacy-write blocks are marked ``# TODO: Delete after Vulnerability_Id retirement``. +The legacy ``Vulnerability_Id`` store is no longer written (entity-only cutover); the legacy table +still exists but is frozen until it is dropped in a follow-up migration. CWE buffering (store_cwes/pending_cwes/reconcile_cwes) is deliberately NOT owned here — it keeps riding the importer flush boundary exactly as before. """ @@ -15,7 +12,6 @@ from django.db import transaction -from dojo.finding.models import Vulnerability_Id from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability @@ -75,25 +71,11 @@ def _write_references(pending): def persist_for_finding(finding, vulnerability_ids, *, delete_existing: bool = True) -> None: """ - Single-finding dual-write core (no buffering). Does NOT touch ``finding.cve`` — the cve sync + Single-finding write core (no buffering). Does NOT touch ``finding.cve`` — the cve sync stays with the caller (e.g. save_vulnerability_ids) exactly as before. """ ids = _clean(vulnerability_ids) with transaction.atomic(): - # --- LEGACY store (byte-for-byte today's behavior) --- - # TODO: Delete after Vulnerability_Id retirement - if delete_existing: - Vulnerability_Id.objects.filter(finding=finding).delete() - Vulnerability_Id.objects.bulk_create( - [ - Vulnerability_Id( - finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid), - ) - for vid in ids - ], - batch_size=1000, - ) - # --- Entity + reference store --- if delete_existing: FindingVulnerabilityReference.objects.filter(finding=finding).delete() _write_references([(finding, ids)]) @@ -102,7 +84,7 @@ def persist_for_finding(finding, vulnerability_ids, *, delete_existing: bool = T class VulnerabilityIdManager: """ - Accumulate/flush dual-writes for a batch of findings (importer path). + Accumulate/flush entity writes for a batch of findings (importer path). Replaces the importer's ``pending_vulnerability_ids`` / ``pending_vuln_id_deletes`` buffers. ``record`` is for new findings (pure insert); ``record_reconcile`` is for reimported findings @@ -118,29 +100,15 @@ def record(self, finding, vulnerability_ids) -> None: self.pending.append((finding, _clean(vulnerability_ids))) def record_reconcile(self, finding, vulnerability_ids) -> None: - """Buffer a changed finding: delete its existing rows/refs, then insert the new ids.""" + """Buffer a changed finding: delete its existing refs, then insert the new ids.""" self.pending_deletes.append(finding.id) self.pending.append((finding, _clean(vulnerability_ids))) def flush(self) -> None: - """Delete stale and bulk-insert the buffered legacy rows + entity refs, then clear.""" + """Delete stale and bulk-insert the buffered entity references, then clear.""" if not self.pending and not self.pending_deletes: return with transaction.atomic(): - # --- LEGACY store (byte-for-byte today's behavior) --- - # TODO: Delete after Vulnerability_Id retirement - if self.pending_deletes: - Vulnerability_Id.objects.filter(finding_id__in=self.pending_deletes).delete() - legacy_rows = [ - Vulnerability_Id( - finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid), - ) - for finding, ids in self.pending - for vid in ids - ] - if legacy_rows: - Vulnerability_Id.objects.bulk_create(legacy_rows, batch_size=1000) - # --- Entity + reference store --- # Delete refs for reimported (changed) findings; new findings have none. if self.pending_deletes: FindingVulnerabilityReference.objects.filter(finding_id__in=self.pending_deletes).delete() diff --git a/dojo/vulnerability/queries.py b/dojo/vulnerability/queries.py index 4a2cfe5d764..2c36902324e 100644 --- a/dojo/vulnerability/queries.py +++ b/dojo/vulnerability/queries.py @@ -1,6 +1,5 @@ import logging -from django.conf import settings from django.db.models import OuterRef, Prefetch try: @@ -11,78 +10,54 @@ def get_auth_filter(key): return None -from dojo.finding.models import Vulnerability_Id from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -# Read seam. The V3_FEATURE_VULNERABILITY_IDS flag is consulted ONLY here and in -# the two Finding methods (vulnerability_ids property, get_vulnerability_ids) — -# nowhere else. Writes are unconditionally dual (see manager.py), so both stores -# hold identical data and this flag selects the READ store only. +# Read helpers over the entity/reference store — the single source of truth. +# (The legacy Vulnerability_Id read path + V3_FEATURE_VULNERABILITY_IDS flag were +# removed in the entity-only cutover; the legacy table is frozen until dropped.) # --------------------------------------------------------------------------- -def use_entity_reads() -> bool: - """True when findings should read vulnerability ids from the entity/reference tables.""" - return getattr(settings, "V3_FEATURE_VULNERABILITY_IDS", False) - - def finding_ids_with_vulnerability_ids(values, *, lookup="in"): """ QuerySet of finding ids whose vulnerability ids match ``values`` (``lookup`` = 'in' or 'exact'). Backs both filter functions in dojo/filters.py and the risk-acceptance exact-match lookup. """ - if use_entity_reads(): - return FindingVulnerabilityReference.objects.filter( - **{f"vulnerability__vulnerability_id__{lookup}": values}, - ).values_list("finding_id", flat=True) - return Vulnerability_Id.objects.filter(**{f"vulnerability_id__{lookup}": values}).values_list( - "finding_id", flat=True, - ) + return FindingVulnerabilityReference.objects.filter( + **{f"vulnerability__vulnerability_id__{lookup}": values}, + ).values_list("finding_id", flat=True) def vulnerability_id_prefetch(prefix=""): """ - Prefetch spec for a finding's vulnerability ids — a string (flag off) or a Prefetch (flag on). + Prefetch spec for a finding's vulnerability id references. ``prefix`` supports nested lookups (e.g. ``"findings__"`` to prefetch through a finding group). """ - if use_entity_reads(): - return Prefetch( - f"{prefix}vulnerability_references", - queryset=FindingVulnerabilityReference.objects.select_related("vulnerability"), - ) - return f"{prefix}vulnerability_id_set" + return Prefetch( + f"{prefix}vulnerability_references", + queryset=FindingVulnerabilityReference.objects.select_related("vulnerability"), + ) def finding_vulnerability_id_strings(finding): """ - Ordered vulnerability id strings for a saved finding from the flag-appropriate store. - - The RAW relation (no cve merge, no dedupe) — backs the wire-frozen v2 serializer field, whose - output must stay byte-identical to reading the legacy vulnerability_id_set. + Ordered vulnerability id strings for a saved finding (raw reference relation — no cve merge, + no dedupe). Honors a prefetched ``vulnerability_references`` to avoid an N+1. """ - if use_entity_reads(): - return [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] - return [row.vulnerability_id for row in finding.vulnerability_id_set.all()] + return [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] def first_vulnerability_id_subquery(): - """ - Subquery selecting a finding's primary (first) vulnerability id string, for annotation. - - Flag on: the reference at ``order == 0`` (== the finding's cve). Flag off: today's variant — - the lowest-PK legacy row. - """ - if use_entity_reads(): - return FindingVulnerabilityReference.objects.filter(finding=OuterRef("pk"), order=0).values( - "vulnerability__vulnerability_id", - )[:1] - return Vulnerability_Id.objects.filter(finding=OuterRef("pk")).order_by("id").values("vulnerability_id")[:1] + """Subquery selecting a finding's primary (order-0, == cve) vulnerability id string.""" + return FindingVulnerabilityReference.objects.filter(finding=OuterRef("pk"), order=0).values( + "vulnerability__vulnerability_id", + )[:1] def get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): diff --git a/unittests/test_apply_finding_template.py b/unittests/test_apply_finding_template.py index 468dacbd36f..44ecac2b6b0 100644 --- a/unittests/test_apply_finding_template.py +++ b/unittests/test_apply_finding_template.py @@ -20,13 +20,13 @@ Engagement, Finding, Finding_Template, + FindingVulnerabilityReference, Notes, Product, Product_Type, System_Settings, Test, Test_Type, - Vulnerability_Id, ) from dojo.test.ui import views as test_views from unittests.dojo_test_case import DojoTestCase, versioned_fixtures @@ -578,7 +578,7 @@ def test_add_finding_from_template_copies_all_fields(self): self.assertEqual(finding.component_version, "1.0.0") # Verify vulnerability IDs were copied - vulnerability_ids = [vid.vulnerability_id for vid in Vulnerability_Id.objects.filter(finding=finding)] + vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in FindingVulnerabilityReference.objects.filter(finding=finding)] self.assertIn("CVE-2023-1234", vulnerability_ids) self.assertIn("CVE-2023-5678", vulnerability_ids) diff --git a/unittests/test_authorization_queries.py b/unittests/test_authorization_queries.py index c1f85a1a419..3d42bf10ec5 100644 --- a/unittests/test_authorization_queries.py +++ b/unittests/test_authorization_queries.py @@ -22,10 +22,10 @@ from dojo.authorization.roles_permissions import Permissions from dojo.endpoint.queries import get_authorized_endpoint_status, get_authorized_endpoints from dojo.engagement.queries import get_authorized_engagements +from dojo.finding.helper import save_vulnerability_ids from dojo.finding.queries import ( get_authorized_findings, get_authorized_findings_for_queryset, - get_authorized_vulnerability_ids, ) from dojo.finding_group.queries import get_authorized_finding_groups from dojo.location.models import LocationFindingReference, LocationProductReference @@ -42,16 +42,17 @@ Engagement, Finding, Finding_Group, + FindingVulnerabilityReference, Product, Product_Type, Test, Test_Type, - Vulnerability_Id, ) from dojo.product.queries import get_authorized_products from dojo.product_type.queries import get_authorized_product_types from dojo.test.queries import get_authorized_tests from dojo.url.models import URL +from dojo.vulnerability.queries import get_authorized_finding_vulnerability_references from .dojo_test_case import DojoTestCase, skip_unless_v2, skip_unless_v3 @@ -238,14 +239,19 @@ def setUpTestData(cls): }, ) - # Create vulnerability IDs - cls.vuln_id_1, _ = Vulnerability_Id.objects.get_or_create( + # Create vulnerability IDs (entity + ordered reference rows) and grab the + # reference objects used later in the authorized-queryset assertions. + save_vulnerability_ids(cls.finding_1, ["CVE-2024-0001"]) + cls.finding_1.save() + cls.vuln_id_1 = FindingVulnerabilityReference.objects.get( finding=cls.finding_1, - vulnerability_id="CVE-2024-0001", + vulnerability__vulnerability_id="CVE-2024-0001", ) - cls.vuln_id_2, _ = Vulnerability_Id.objects.get_or_create( + save_vulnerability_ids(cls.finding_2, ["CVE-2024-0002"]) + cls.finding_2.save() + cls.vuln_id_2 = FindingVulnerabilityReference.objects.get( finding=cls.finding_2, - vulnerability_id="CVE-2024-0002", + vulnerability__vulnerability_id="CVE-2024-0002", ) if settings.V3_FEATURE_LOCATIONS: @@ -363,23 +369,25 @@ def test_none_user_returns_empty(self): class TestGetAuthorizedVulnerabilityIds(AuthorizationQueriesTestBase): - """Tests for get_authorized_vulnerability_ids()""" + """Tests for get_authorized_finding_vulnerability_references()""" def test_superuser_gets_all_vulnerability_ids(self): - """Superuser should get all vulnerability IDs""" - vuln_ids = get_authorized_vulnerability_ids(Permissions.Finding_View, user=self.superuser) + """Superuser should get all vulnerability id references""" + vuln_ids = get_authorized_finding_vulnerability_references(Permissions.Finding_View, user=self.superuser) self.assertIn(self.vuln_id_1, vuln_ids) self.assertIn(self.vuln_id_2, vuln_ids) def test_user_no_permissions_gets_empty(self): - """User with no permissions should not get test vulnerability IDs""" - vuln_ids = get_authorized_vulnerability_ids(Permissions.Finding_View, user=self.user_no_perms) + """User with no permissions should not get test vulnerability id references""" + vuln_ids = get_authorized_finding_vulnerability_references(Permissions.Finding_View, user=self.user_no_perms) self.assertNotIn(self.vuln_id_1, vuln_ids) self.assertNotIn(self.vuln_id_2, vuln_ids) def test_user_product_member_gets_product_vulnerability_ids(self): - """User with product membership should get only that product's vulnerability IDs""" - vuln_ids = get_authorized_vulnerability_ids(Permissions.Finding_View, user=self.user_product_member) + """User with product membership should get only that product's vulnerability id references""" + vuln_ids = get_authorized_finding_vulnerability_references( + Permissions.Finding_View, user=self.user_product_member, + ) self.assertIn(self.vuln_id_1, vuln_ids) self.assertNotIn(self.vuln_id_2, vuln_ids) diff --git a/unittests/test_authorization_queryset_coverage.py b/unittests/test_authorization_queryset_coverage.py index a894ec0d219..84b9be54932 100644 --- a/unittests/test_authorization_queryset_coverage.py +++ b/unittests/test_authorization_queryset_coverage.py @@ -15,11 +15,9 @@ from dojo.finding.queries import ( get_authorized_findings, get_authorized_findings_for_queryset, - get_authorized_vulnerability_ids, - get_authorized_vulnerability_ids_for_queryset, ) from dojo.finding_group.queries import get_authorized_finding_groups -from dojo.models import Dojo_User, Endpoint, Finding, Test, Vulnerability_Id +from dojo.models import Dojo_User, Endpoint, Finding, Test from dojo.product.queries import ( get_authorized_app_analysis, get_authorized_engagement_presets, @@ -30,6 +28,7 @@ from dojo.product_type.queries import get_authorized_product_types from dojo.risk_acceptance.queries import get_authorized_risk_acceptances from dojo.test.queries import get_authorized_test_imports, get_authorized_tests +from dojo.vulnerability.queries import get_authorized_finding_vulnerability_references from .dojo_test_case import DojoTestCase, skip_unless_v2, versioned_fixtures @@ -65,10 +64,7 @@ def _cases(self): ("findings_for_queryset", lambda: get_authorized_findings_for_queryset("view", Finding.objects.all()), "test__engagement__product__id"), - ("vulnerability_ids", lambda: get_authorized_vulnerability_ids("view"), - "finding__test__engagement__product__id"), - ("vulnerability_ids_for_queryset", - lambda: get_authorized_vulnerability_ids_for_queryset("view", Vulnerability_Id.objects.all()), + ("vulnerability_references", lambda: get_authorized_finding_vulnerability_references("view"), "finding__test__engagement__product__id"), ("app_analysis", lambda: get_authorized_app_analysis("view"), "product__id"), ("languages", lambda: get_authorized_languages("view"), "product__id"), diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py index e883f6c6613..4e5766dbaf0 100644 --- a/unittests/test_importers_importer.py +++ b/unittests/test_importers_importer.py @@ -14,12 +14,12 @@ Development_Environment, Engagement, Finding, + FindingVulnerabilityReference, Product, Product_Type, Test, Test_Type, User, - Vulnerability_Id, ) from dojo.tools.gitlab_sast.parser import GitlabSastParser from dojo.tools.sarif.parser import SarifParser @@ -1089,8 +1089,8 @@ def test_clear_vulnerability_ids_on_empty_list(self): # Verify IDs are cleared self.assertEqual(0, len(finding.vulnerability_ids)) self.assertEqual(None, finding.cve) - # Verify no Vulnerability_Id objects exist for this finding - self.assertEqual(0, Vulnerability_Id.objects.filter(finding=finding).count()) + # Verify no vulnerability id references exist for this finding + self.assertEqual(0, FindingVulnerabilityReference.objects.filter(finding=finding).count()) finding.delete() def test_change_vulnerability_ids_on_reimport(self): @@ -1128,8 +1128,8 @@ def test_change_vulnerability_ids_on_reimport(self): self.assertEqual("CVE-2021-9999", finding.vulnerability_ids[0]) self.assertEqual("GHSA-xxxx-yyyy", finding.vulnerability_ids[1]) self.assertEqual("CVE-2021-9999", finding.cve) - # Verify only new Vulnerability_Id objects exist - vuln_ids = list(Vulnerability_Id.objects.filter(finding=finding).values_list("vulnerability_id", flat=True)) + # Verify only new vulnerability id references exist + vuln_ids = list(FindingVulnerabilityReference.objects.filter(finding=finding).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(set(new_vulnerability_ids), set(vuln_ids)) finding.delete() @@ -1140,23 +1140,19 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): # finding_a: IDs change (CVE-A → CVE-B) finding_a = Finding(test=self.test, reporter=self.testuser) finding_a.save() - Vulnerability_Id.objects.create(finding=finding_a, vulnerability_id="CVE-A-OLD") - finding_a.cve = "CVE-A-OLD" + save_vulnerability_ids(finding_a, ["CVE-A-OLD"]) finding_a.save() # finding_b: IDs change (CVE-B1, CVE-B2 → CVE-B-NEW) finding_b = Finding(test=self.test, reporter=self.testuser) finding_b.save() - Vulnerability_Id.objects.create(finding=finding_b, vulnerability_id="CVE-B1") - Vulnerability_Id.objects.create(finding=finding_b, vulnerability_id="CVE-B2") - finding_b.cve = "CVE-B1" + save_vulnerability_ids(finding_b, ["CVE-B1", "CVE-B2"]) finding_b.save() # finding_c: IDs unchanged — should not appear in delete/insert buffers finding_c = Finding(test=self.test, reporter=self.testuser) finding_c.save() - Vulnerability_Id.objects.create(finding=finding_c, vulnerability_id="CVE-C-SAME") - finding_c.cve = "CVE-C-SAME" + save_vulnerability_ids(finding_c, ["CVE-C-SAME"]) finding_c.save() finding_a.unsaved_vulnerability_ids = ["CVE-A-NEW"] @@ -1176,8 +1172,8 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): self.assertEqual(2, sum(len(ids) for _, ids in manager.pending)) # Old IDs still in DB (not yet deleted) - self.assertEqual(1, Vulnerability_Id.objects.filter(finding=finding_a).count()) - self.assertEqual(2, Vulnerability_Id.objects.filter(finding=finding_b).count()) + self.assertEqual(1, FindingVulnerabilityReference.objects.filter(finding=finding_a).count()) + self.assertEqual(2, FindingVulnerabilityReference.objects.filter(finding=finding_b).count()) reimporter.flush_vulnerability_ids() @@ -1186,17 +1182,17 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): self.assertEqual([], reimporter.vulnerability_id_manager.pending) # finding_a: old deleted, new inserted - vuln_ids_a = list(Vulnerability_Id.objects.filter(finding=finding_a).values_list("vulnerability_id", flat=True)) + vuln_ids_a = list(FindingVulnerabilityReference.objects.filter(finding=finding_a).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(["CVE-A-NEW"], vuln_ids_a) self.assertEqual("CVE-A-NEW", finding_a.cve) # finding_b: both old deleted, new inserted - vuln_ids_b = list(Vulnerability_Id.objects.filter(finding=finding_b).values_list("vulnerability_id", flat=True)) + vuln_ids_b = list(FindingVulnerabilityReference.objects.filter(finding=finding_b).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(["CVE-B-NEW"], vuln_ids_b) self.assertEqual("CVE-B-NEW", finding_b.cve) # finding_c: unchanged — IDs untouched - vuln_ids_c = list(Vulnerability_Id.objects.filter(finding=finding_c).values_list("vulnerability_id", flat=True)) + vuln_ids_c = list(FindingVulnerabilityReference.objects.filter(finding=finding_c).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(["CVE-C-SAME"], vuln_ids_c) finding_a.delete() @@ -1209,8 +1205,7 @@ def test_reconcile_vulnerability_ids_unchanged_no_db_write(self): finding = Finding(test=self.test, reporter=self.testuser) finding.save() - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") - finding.cve = "CVE-2020-1234" + save_vulnerability_ids(finding, ["CVE-2020-1234"]) finding.save() finding.unsaved_vulnerability_ids = ["CVE-2020-1234"] diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index eedc01943f9..01c8fdcdf2d 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -40,12 +40,12 @@ Endpoint_Status, Engagement, Finding, + FindingVulnerabilityReference, Product, Product_Type, Test, User, UserContactInfo, - Vulnerability_Id, ) from dojo.tools.stackhawk.parser import StackHawkParser @@ -77,7 +77,7 @@ def setUp(self): # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count. # But if we run the test in isolation, the ContentType ids will not be cached and will result in more queries. # By warming up the cache here, these queries are executed before we start counting queries - for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User, Test, Vulnerability_Id]: + for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User, Test, FindingVulnerabilityReference]: ContentType.objects.get_for_model(model) @contextmanager diff --git a/unittests/test_migrations.py b/unittests/test_migrations.py index e6a0481891d..ec2f3116d0f 100644 --- a/unittests/test_migrations.py +++ b/unittests/test_migrations.py @@ -2,25 +2,9 @@ from unittest import skip from django.contrib.auth import get_user_model -from django.test import TestCase from django.utils import timezone from django_test_migrations.contrib.unittest_case import MigratorTestCase -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type -from dojo.models import ( - Development_Environment, - Engagement, - Finding, - FindingVulnerabilityReference, - Product, - Product_Type, - Test, - Test_Type, - Vulnerability, - Vulnerability_Id, -) -from dojo.vulnerability.backfill import run_backfill - @skip("Outdated - this class was testing some version of migration; it is not needed anymore") class TestOptiEndpointStatus(MigratorTestCase): @@ -177,162 +161,3 @@ def test_after_migration(self): ) self.assertEqual(eps.all().count(), 1, ep) self.assertIsInstance(eps.all().first(), Endpoint_Status) - - -class TestVulnerabilityIdBackfill(TestCase): - - """ - dojo.vulnerability.backfill.run_backfill (the routine migration 0286 delegates to) turns - legacy dojo_vulnerability_id rows + dojo_finding.cve into Vulnerability entities and ordered - FindingVulnerabilityReference rows (order 0 == cve). Covers ordered ids, cve-drift (cve is not - the lowest-PK row), case-variant ids (2 distinct entities), cve-only findings (no legacy rows => - NO reference, so the reference set mirrors the legacy row set and the hash stays byte-identical; - the cve remains an orphan registry entity), null-cve findings, and an idempotent re-run. - - A plain TestCase rather than MigratorTestCase: django_test_migrations replays the full squashed - migration chain from a wiped schema in setUp, which is unusable in this codebase (the replay - collides on tables such as dojo_cred_user whose model no longer matches the squash) — the very - reason the only other MigratorTestCase here is skipped. run_backfill IS what 0286 runs, and the - migration's application is separately covered by the create-db and dev-DB migrate runs. - """ - - @classmethod - def setUpTestData(cls): - now = timezone.now() - prod_type = Product_Type.objects.create(name="vulnid-pt") - # description / scan_type / environment are required under full model validation - # (base save full_clean when V3_FEATURE_LOCATIONS is on — the CI "(true)" matrix leg). - product = Product.objects.create(prod_type=prod_type, name="vulnid-prod", description="vuln-id backfill test") - engagement = Engagement.objects.create(product=product, target_start=now, target_end=now) - test_type = Test_Type.objects.create(name="vulnid-tt") - environment = Development_Environment.objects.get_or_create(name="Development")[0] - test = Test.objects.create( - engagement=engagement, - test_type=test_type, - scan_type="vulnid-tt", - environment=environment, - target_start=now, - target_end=now, - ) - user = get_user_model().objects.create(username="vulnid-user") - - def make_finding(title, cve): - return Finding.objects.create( - test=test, - reporter=user, - title=title, - description=title, - severity="High", - numerical_severity="S0", - active=True, - verified=False, - cve=cve, - ) - - def add_legacy(finding, vulnerability_id): - # Real save() stamps vulnerability_id_type, mirroring production legacy rows. - Vulnerability_Id.objects.create(finding=finding, vulnerability_id=vulnerability_id) - - # A: ordered ids, cve == first (lowest-PK) row. - cls.finding_a = make_finding("A", "CVE-A1") - for vid in ("CVE-A1", "CVE-A2", "CVE-A3"): - add_legacy(cls.finding_a, vid) - - # B: cve-drift — rows created B1,B2,B3 (ascending PK) but cve == CVE-B2 (middle row). - cls.finding_b = make_finding("B", "CVE-B2") - for vid in ("CVE-B1", "CVE-B2", "CVE-B3"): - add_legacy(cls.finding_b, vid) - - # C: case-variant ids — CVE-C1 and cve-c1 are distinct exact-cased strings => 2 entities. - cls.finding_c = make_finding("C", "CVE-C1") - for vid in ("CVE-C1", "cve-c1"): - add_legacy(cls.finding_c, vid) - - # D: cve-only — cve set, NO legacy rows => NO reference (mirror the empty legacy row set for - # hash parity); CVE-D1 still becomes an orphan registry entity via step 2. - cls.finding_d = make_finding("D", "CVE-D1") - - # E: legacy rows but NULL cve => order by PK, no cve-only ref. - cls.finding_e = make_finding("E", None) - for vid in ("CVE-E1", "CVE-E2"): - add_legacy(cls.finding_e, vid) - - def _order_map(self, finding): - return { - ref.order: ref.vulnerability.vulnerability_id - for ref in FindingVulnerabilityReference.objects.filter(finding=finding) - } - - def test_backfill(self): - run_backfill() - - expected_entities = { - "CVE-A1", - "CVE-A2", - "CVE-A3", - "CVE-B1", - "CVE-B2", - "CVE-B3", - "CVE-C1", - "cve-c1", - "CVE-D1", - "CVE-E1", - "CVE-E2", - } - - with self.subTest("entity extraction (exact-cased, case-variant = 2 entities)"): - self.assertEqual( - set(Vulnerability.objects.values_list("vulnerability_id", flat=True)), - expected_entities, - ) - - with self.subTest("entity type carried / stamped from prefix"): - for entity in Vulnerability.objects.all(): - self.assertEqual( - entity.vulnerability_id_type, - resolve_vulnerability_id_type(entity.vulnerability_id), - ) - - with self.subTest("A: ordered ids, cve first"): - self.assertEqual(self._order_map(self.finding_a), {0: "CVE-A1", 1: "CVE-A2", 2: "CVE-A3"}) - - with self.subTest("B: cve-drift resolves cve to order 0, rest by PK asc"): - self.assertEqual(self._order_map(self.finding_b), {0: "CVE-B2", 1: "CVE-B1", 2: "CVE-B3"}) - - with self.subTest("C: case-variant ids both referenced, exact cve at order 0"): - self.assertEqual(self._order_map(self.finding_c), {0: "CVE-C1", 1: "cve-c1"}) - - with self.subTest("D: cve-only finding gets NO reference (parity: mirrors empty legacy set)"): - self.assertEqual(self._order_map(self.finding_d), {}) - # The cve is still catalogued as an orphan registry entity (never referenced). - cve_d = Vulnerability.objects.get(vulnerability_id="CVE-D1") - self.assertFalse(cve_d.finding_references.exists()) - - with self.subTest("E: null-cve finding ordered by PK, no cve-only ref"): - self.assertEqual(self._order_map(self.finding_e), {0: "CVE-E1", 1: "CVE-E2"}) - - with self.subTest("pair sets: the reference set EXACTLY mirrors the legacy row set"): - legacy_pairs = set(Vulnerability_Id.objects.values_list("finding_id", "vulnerability_id")) - new_pairs = { - (ref.finding_id, ref.vulnerability.vulnerability_id) - for ref in FindingVulnerabilityReference.objects.all() - } - # No fabricated (cve-only) references: the two stores hold identical (finding, id) pairs, - # which is exactly what keeps get_vulnerability_ids() byte-identical across the flag. - self.assertEqual(new_pairs, legacy_pairs) - - with self.subTest("idempotency: re-running the backfill changes nothing"): - entities_before = Vulnerability.objects.count() - refs_before = FindingVulnerabilityReference.objects.count() - counts = run_backfill() - self.assertEqual(Vulnerability.objects.count(), entities_before) - self.assertEqual(FindingVulnerabilityReference.objects.count(), refs_before) - self.assertEqual( - counts, - { - "entities_from_legacy": 0, - "entities_from_cve": 0, - "types_stamped": 0, - "references_from_legacy": 0, - }, - ) diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index dcf50c51f2c..7103c71b2f3 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -597,10 +597,12 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # Multiple-CWEs feature: +2 import / +2 reimport-no-change (Finding_CWE # store + bulk flush) and +10 reimport-with-new (per-finding reconcile reads # existing Finding_CWE rows for each changed finding). - # Vulnerability_Id entity dual-write (unconditional): +2 import / +12 - # reimport-no-change / +6 reimport-with-new queries (Vulnerability entity + - # FindingVulnerabilityReference bulk writes alongside the legacy rows; batched, - # not per-finding). + # Vulnerability id writes (entity-only cutover): only the Vulnerability entity + + # FindingVulnerabilityReference bulk writes remain (batched, not per-finding). The + # legacy Vulnerability_Id dual-write was removed, so these counts drop by the legacy + # delete+bulk_insert per flush. NOTE: recalibrate the constants below under OSS CI + # (watson enabled) — they cannot be measured in the Pro test env, which force-disables + # watson so this fixture-backed test does not run there. EXPECTED_ZAP_IMPORT_V2 = 296 EXPECTED_ZAP_IMPORT_V3 = 320 EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 91 diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index cc3e09716dd..36b60339504 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -1,20 +1,18 @@ """ -Unconditional dual-write regression tests for the vulnerability-id entity store. +Regression tests for the vulnerability-id entity store (entity-only, post-cutover). Fixture-free (no dojo_testdata.json) so they run in both the OSS CI DB and the Pro container. -They assert the NEW behavior that the legacy Vulnerability_Id tests do not: that persist_for_finding -and VulnerabilityIdManager write the Vulnerability entity + ordered FindingVulnerabilityReference -rows alongside the legacy rows, keep order 0 == first id, reconcile both stores, and never delete -entity rows (orphans are retained). +They assert that persist_for_finding and VulnerabilityIdManager write the Vulnerability entity + +ordered FindingVulnerabilityReference rows (order 0 == cve) as the single source of truth, that +entity rows are never deleted (orphans are retained), and that the read surfaces (hash input, +property, v2 wire, filter helper) are correct. The legacy Vulnerability_Id store has been dropped. """ import pathlib -from io import StringIO from django.contrib.auth import get_user_model -from django.core.management import call_command from django.db import IntegrityError, transaction -from django.test import TestCase, override_settings +from django.test import TestCase from django.utils.timezone import now from dojo.finding.api.serializer import VulnerabilityIdsField @@ -29,16 +27,15 @@ Test, Test_Type, Vulnerability, - Vulnerability_Id, ) from dojo.tools.factory import get_parser from dojo.vulnerability.manager import VulnerabilityIdManager, persist_for_finding from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids -class VulnerabilityIdDualWriteTestCase(TestCase): +class VulnerabilityEntityTestCase(TestCase): - """Shared minimal Product -> Engagement -> Test scaffold for dual-write tests.""" + """Shared minimal Product -> Engagement -> Test scaffold for entity-store tests.""" @classmethod def setUpTestData(cls): @@ -72,9 +69,6 @@ def _make_finding(self, title): numerical_severity="S0", ) - def _legacy(self, finding): - return sorted(Vulnerability_Id.objects.filter(finding=finding).values_list("vulnerability_id", flat=True)) - def _refs(self, finding): return { ref.order: ref.vulnerability.vulnerability_id @@ -82,14 +76,12 @@ def _refs(self, finding): } -class TestPersistForFinding(VulnerabilityIdDualWriteTestCase): - def test_dual_write_and_reconcile(self): +class TestPersistForFinding(VulnerabilityEntityTestCase): + def test_write_and_reconcile(self): finding = self._make_finding("persist") persist_for_finding(finding, ["CVE-2021-1", "CVE-2021-2", "CVE-2021-2"], delete_existing=False) - with self.subTest("legacy rows written (deduped)"): - self.assertEqual(self._legacy(finding), ["CVE-2021-1", "CVE-2021-2"]) with self.subTest("entity rows created"): self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-1").exists()) self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-2").exists()) @@ -99,34 +91,30 @@ def test_dual_write_and_reconcile(self): # Reconcile with changed ids. persist_for_finding(finding, ["CVE-2021-2", "CVE-2021-9"], delete_existing=True) - with self.subTest("legacy reconciled"): - self.assertEqual(self._legacy(finding), ["CVE-2021-2", "CVE-2021-9"]) with self.subTest("references reconciled with new order"): self.assertEqual(self._refs(finding), {0: "CVE-2021-2", 1: "CVE-2021-9"}) with self.subTest("dropped id's entity is retained (orphan registry)"): self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-1").exists()) - def test_empty_clears_both_stores(self): + def test_empty_clears_references(self): finding = self._make_finding("persist-empty") persist_for_finding(finding, ["CVE-2022-1"], delete_existing=False) self.assertEqual(self._refs(finding), {0: "CVE-2022-1"}) persist_for_finding(finding, [], delete_existing=True) - self.assertEqual(self._legacy(finding), []) self.assertEqual(self._refs(finding), {}) -class TestSaveVulnerabilityIdsIntegration(VulnerabilityIdDualWriteTestCase): - def test_save_vulnerability_ids_dual_writes_and_syncs_cve(self): +class TestSaveVulnerabilityIdsIntegration(VulnerabilityEntityTestCase): + def test_save_vulnerability_ids_writes_references_and_syncs_cve(self): finding = self._make_finding("helper") save_vulnerability_ids(finding, ["CVE-2023-1", "CVE-2023-2"], delete_existing=False) - self.assertEqual(self._legacy(finding), ["CVE-2023-1", "CVE-2023-2"]) self.assertEqual(self._refs(finding), {0: "CVE-2023-1", 1: "CVE-2023-2"}) self.assertEqual(finding.cve, "CVE-2023-1") -class TestVulnerabilityIdManager(VulnerabilityIdDualWriteTestCase): +class TestVulnerabilityIdManager(VulnerabilityEntityTestCase): def test_batch_record_and_reconcile_flush(self): finding_new = self._make_finding("mgr-new") finding_reimport = self._make_finding("mgr-reimport") @@ -138,23 +126,20 @@ def test_batch_record_and_reconcile_flush(self): manager.record_reconcile(finding_reimport, ["CVE-OLD-2", "CVE-NEW-3"]) manager.flush() - with self.subTest("new finding dual-written"): - self.assertEqual(self._legacy(finding_new), ["CVE-NEW-1", "CVE-NEW-2"]) + with self.subTest("new finding references written"): self.assertEqual(self._refs(finding_new), {0: "CVE-NEW-1", 1: "CVE-NEW-2"}) - with self.subTest("reimported finding reconciled in both stores"): - self.assertEqual(self._legacy(finding_reimport), ["CVE-NEW-3", "CVE-OLD-2"]) + with self.subTest("reimported finding references reconciled"): self.assertEqual(self._refs(finding_reimport), {0: "CVE-OLD-2", 1: "CVE-NEW-3"}) with self.subTest("buffers cleared after flush"): self.assertEqual(manager.pending, []) self.assertEqual(manager.pending_deletes, []) -class TestReadSeamFlagParity(VulnerabilityIdDualWriteTestCase): +class TestReadCorrectness(VulnerabilityEntityTestCase): """ - The read seam produces byte-identical results in both flag states for a seam-written finding - (dual-write keeps both stores in sync). Covers the v2 wire shape, the hash input - (get_vulnerability_ids), the vulnerability_ids property, and the filter seam. + The read surfaces over the entity store: the v2 wire shape, the hash input + (get_vulnerability_ids), the vulnerability_ids property, and the filter helper. """ @classmethod @@ -174,211 +159,27 @@ def _fresh(self): # Fresh instance each read: vulnerability_ids is a cached_property. return Finding.objects.get(pk=self.finding.pk) - def test_v2_wire_shape_parity(self): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off = VulnerabilityIdsField().to_representation(self._fresh()) - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on = VulnerabilityIdsField().to_representation(self._fresh()) - self.assertEqual(off, on) + def test_v2_wire_shape(self): self.assertEqual( - off, + VulnerabilityIdsField().to_representation(self._fresh()), [{"vulnerability_id": "CVE-P-1"}, {"vulnerability_id": "CVE-P-2"}, {"vulnerability_id": "CVE-P-3"}], ) - def test_hash_input_parity(self): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off = self._fresh().get_vulnerability_ids() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on = self._fresh().get_vulnerability_ids() - self.assertEqual(off, on) - - def test_property_parity(self): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off = self._fresh().vulnerability_ids - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on = self._fresh().vulnerability_ids - self.assertEqual(off, on) - self.assertEqual(off, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) - - def test_filter_seam_parity(self): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off = set(finding_ids_with_vulnerability_ids(["CVE-P-2"], lookup="in")) - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on = set(finding_ids_with_vulnerability_ids(["CVE-P-2"], lookup="in")) - self.assertEqual(off, on) - self.assertEqual(off, {self.finding.pk}) - # Exact-match lane too. - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - self.assertEqual( - set(finding_ids_with_vulnerability_ids("CVE-P-1", lookup="exact")), - {self.finding.pk}, - ) - - -class TestVulnerabilityIdCommands(VulnerabilityIdDualWriteTestCase): - - """ - migrate_vulnerability_ids backfills entities/references from legacy rows (idempotently); - verify_vulnerability_ids passes when the stores agree and exits nonzero on drift. - """ - - def test_migrate_command_backfills_idempotently(self): - finding = self._make_finding("cmd-migrate") - finding.cve = "CVE-CMD-1" - finding.save() - # Legacy rows WITHOUT entity references (bypassing the dual-write seam, i.e. pre-migration). - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-CMD-1") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-CMD-2") - self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) - - call_command("migrate_vulnerability_ids") - - # cve-match lands at order 0. - self.assertEqual(self._refs(finding), {0: "CVE-CMD-1", 1: "CVE-CMD-2"}) - self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-CMD-1").exists()) - - # Idempotent re-run: no change. - call_command("migrate_vulnerability_ids") - self.assertEqual(self._refs(finding), {0: "CVE-CMD-1", 1: "CVE-CMD-2"}) - self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 2) - - def test_verify_command_passes_when_consistent(self): - finding = self._make_finding("cmd-verify-ok") - save_vulnerability_ids(finding, ["CVE-V-1", "CVE-V-2"]) - finding.save() - # Consistent stores => no SystemExit. - call_command("verify_vulnerability_ids") - - def test_verify_command_fails_on_discrepancy(self): - finding = self._make_finding("cmd-verify-bad") - finding.cve = "CVE-BAD-1" - finding.save() - # Legacy row with no matching entity reference => a discrepancy. - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-BAD-1") - with self.assertRaises(SystemExit): - call_command("verify_vulnerability_ids") - - def test_verify_warns_not_fails_when_cve_absent_from_ids(self): - # Anomalous pre-upgrade data: the finding's cve is set to a value that is NOT among its - # legacy vulnerability-id rows. The backfill cannot make order-0 == cve without injecting - # cve into the reference set, which would diverge from the legacy id set and break the hash - # invariant. verify must therefore treat order-0 != cve as a NON-FATAL warning here (not a - # hard discrepancy), while parity between the two stores still holds. - finding = self._make_finding("cmd-verify-cve-absent") - finding.cve = "CVE-ABSENT-9" - finding.save() - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-PRESENT-1") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-PRESENT-2") - - call_command("migrate_vulnerability_ids") - - # order-0 is a legacy row, NOT the cve (which is not among the ids at all). - refs = self._refs(finding) - self.assertNotEqual(refs[0], finding.cve) - self.assertNotIn(finding.cve, refs.values()) - - # verify does NOT raise (warning path), and reports the finding as a non-fatal warning. - out = StringIO() - call_command("verify_vulnerability_ids", stdout=out) - output = out.getvalue() - self.assertIn("no discrepancies", output) - self.assertIn("warnings (non-fatal)", output) - self.assertIn("CVE-ABSENT-9", output) - - # Hash parity across the flag still holds (the reference set == the legacy id set). - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off_hash = Finding.objects.get(pk=finding.pk).get_vulnerability_ids() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on_hash = Finding.objects.get(pk=finding.pk).get_vulnerability_ids() - self.assertEqual(off_hash, on_hash) - - -class TestUpgradeReadParity(VulnerabilityIdDualWriteTestCase): - - """ - Upgrade path: legacy Vulnerability_Id rows written the OLD way (before the entity existed) and - then backfilled by migrate_vulnerability_ids must read back IDENTICALLY in both flag states. - - Distinct from TestReadSeamFlagParity, which seeds through the dual-write seam (so both stores - are trivially in sync). Here only the LEGACY rows are written — the true pre-upgrade state — - then backfilled, then flag-off (legacy read) is compared against flag-on (entity read). This is - the scenario a real deployment actually hits when the flag is first turned on after a migration. - """ - - def _seed_legacy_only(self, finding, cve, ordered_ids): - """ - Write legacy rows directly (creation order == PK order), set the primary cve, and assert - the pre-migration state (no entity references yet) — i.e. exactly what pre-upgrade data looks - like. - """ - finding.cve = cve - finding.save() - for vid in ordered_ids: - Vulnerability_Id.objects.create(finding=finding, vulnerability_id=vid) - self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) + def test_hash_input(self): + self.assertEqual(self._fresh().get_vulnerability_ids(), "CVE-P-1CVE-P-2CVE-P-3") - def _fresh(self, pk): - # Fresh instance per read: vulnerability_ids is a cached_property. - return Finding.objects.get(pk=pk) + def test_property(self): + self.assertEqual(self._fresh().vulnerability_ids, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) - def _reads(self, pk): - return ( - self._fresh(pk).get_vulnerability_ids(), - self._fresh(pk).vulnerability_ids, - [r["vulnerability_id"] for r in VulnerabilityIdsField().to_representation(self._fresh(pk))], + def test_filter_helper(self): + self.assertEqual(set(finding_ids_with_vulnerability_ids(["CVE-P-2"], lookup="in")), {self.finding.pk}) + self.assertEqual( + set(finding_ids_with_vulnerability_ids("CVE-P-1", lookup="exact")), + {self.finding.pk}, ) - def test_backfilled_normal_data_reads_identically(self): - # Seam/historical invariant: the primary cve IS the first-created (lowest-PK) legacy row. - finding = self._make_finding("upgrade-normal") - self._seed_legacy_only(finding, cve="CVE-U-1", ordered_ids=["CVE-U-1", "CVE-U-2", "CVE-U-3"]) - - call_command("migrate_vulnerability_ids") - self.assertEqual(self._refs(finding)[0], "CVE-U-1") # cve at order 0 - - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off_hash, off_prop, off_wire = self._reads(finding.pk) - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on_hash, on_prop, on_wire = self._reads(finding.pk) - - # Full parity: hash input, the merged property, AND the raw v2 wire order. - self.assertEqual(off_hash, on_hash) - self.assertEqual(off_prop, on_prop) - self.assertEqual(off_wire, on_wire) - self.assertEqual(off_prop, ["CVE-U-1", "CVE-U-2", "CVE-U-3"]) - self.assertEqual(off_wire, ["CVE-U-1", "CVE-U-2", "CVE-U-3"]) - - def test_backfilled_cve_not_first_row_preserves_hash_and_property(self): - # Anomalous pre-upgrade data: the primary cve is NOT the first-created legacy row (e.g. the - # cve field was edited independently of the id rows historically). The hash (sorted) and the - # vulnerability_ids property (cve-merged + deduped) are order-insensitive by construction and - # stay byte-identical across flag states. Only the RAW v2 wire ORDER differs — legacy reads - # PK-ascending, the entity reads cve-first — carrying the SAME set in a different order. This - # divergence is bounded to this anomalous shape (never produced by save_vulnerability_ids, - # which always makes cve == ids[0] == the lowest-PK row) and is asserted here so any future - # change to it is deliberate, not silent. - finding = self._make_finding("upgrade-cve-mid") - self._seed_legacy_only(finding, cve="CVE-M-3", ordered_ids=["CVE-M-1", "CVE-M-2", "CVE-M-3"]) - - call_command("migrate_vulnerability_ids") - self.assertEqual(self._refs(finding)[0], "CVE-M-3") # cve at order 0 despite being last-created - - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off_hash, off_prop, off_wire = self._reads(finding.pk) - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on_hash, on_prop, on_wire = self._reads(finding.pk) - - # Hash + property: byte-identical across the flag. - self.assertEqual(off_hash, on_hash) - self.assertEqual(off_prop, on_prop) - self.assertEqual(off_prop, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # cve first, then PK-asc rest - # Raw v2 wire: same set, different order (the documented, bounded divergence). - self.assertEqual(off_wire, ["CVE-M-1", "CVE-M-2", "CVE-M-3"]) # legacy PK order - self.assertEqual(on_wire, ["CVE-M-3", "CVE-M-1", "CVE-M-2"]) # entity cve-first order - self.assertEqual(set(off_wire), set(on_wire)) - - -class TestReferenceOrderUniqueness(VulnerabilityIdDualWriteTestCase): + +class TestReferenceOrderUniqueness(VulnerabilityEntityTestCase): """ The unique(finding, order) constraint structurally guarantees a single primary id per @@ -403,92 +204,33 @@ def test_distinct_orders_allowed(self): self.assertEqual(self._refs(finding), {0: "CVE-ORD-C", 1: "CVE-ORD-D"}) -class TestLegacyStoreIndependence(VulnerabilityIdDualWriteTestCase): +class TestEntityOnlyLifecycle(VulnerabilityEntityTestCase): """ - CUTOVER SAFETY PROOF: entity reads never touch the legacy Vulnerability_Id store. - - This is THE property that makes retiring the dual-write (a one-way conversion) safe. With the - read flag ON, deleting a finding's legacy rows while leaving its entity references intact must - not change ANY read surface. If poisoning legacy changes nothing, nothing reads it, so the - legacy write can be dropped without touching behavior. Runs flag-ON only, because that is the - post-cutover read path. All read surfaces are checked in one pass (subTest) so every dependency - on legacy surfaces at once rather than one failure at a time. + End-to-end entity-only: write, reconcile, and read a finding's vulnerability ids, with the + legacy Vulnerability_Id store never written. """ - def _surfaces(self, fid): - # Fresh instance per read: vulnerability_ids is a cached_property. - return { - "get_vulnerability_ids (hash input)": Finding.objects.get(pk=fid).get_vulnerability_ids(), - "compute_hash_code": Finding.objects.get(pk=fid).compute_hash_code(), - "vulnerability_ids property": sorted(Finding.objects.get(pk=fid).vulnerability_ids), - "v2 wire": sorted( - r["vulnerability_id"] for r in VulnerabilityIdsField().to_representation(Finding.objects.get(pk=fid)) - ), - "filter __in finds finding": fid - in set(finding_ids_with_vulnerability_ids(["CVE-IND-1", "CVE-IND-2", "CVE-IND-3"])), - "filter __exact finds finding": fid in set(finding_ids_with_vulnerability_ids("CVE-IND-2", lookup="exact")), - } - - @override_settings(V3_FEATURE_VULNERABILITY_IDS=True) - def test_reads_are_independent_of_legacy_store(self): - finding = self._make_finding("poison") - save_vulnerability_ids(finding, ["CVE-IND-1", "CVE-IND-2", "CVE-IND-3"]) - finding.save() - fid = finding.pk - - control = self._surfaces(fid) - # Sanity: the finding is genuinely discoverable through the seam before poisoning. - self.assertEqual(control["get_vulnerability_ids (hash input)"], "CVE-IND-1CVE-IND-2CVE-IND-3") - self.assertTrue(control["filter __in finds finding"]) - - # Poison: delete ALL legacy rows for this finding; entity references stay intact. - deleted, _ = Vulnerability_Id.objects.filter(finding_id=fid).delete() - self.assertEqual(deleted, 3) - self.assertEqual(Vulnerability_Id.objects.filter(finding_id=fid).count(), 0) - self.assertEqual(FindingVulnerabilityReference.objects.filter(finding_id=fid).count(), 3) - - after = self._surfaces(fid) - for surface, value in control.items(): - with self.subTest(surface=surface): - self.assertEqual( - after[surface], - value, - f"read surface {surface!r} changed after poisoning the legacy store -> it still reads legacy", - ) - - @override_settings(V3_FEATURE_VULNERABILITY_IDS=True) - def test_full_lifecycle_with_zero_legacy_rows(self): - # Model the post-cutover world directly: write through the seam, immediately drop the legacy - # rows (so only entity refs exist, as if the legacy write had been removed), and exercise the - # lifecycle. Reads must be correct with NO legacy rows ever backing them. + def test_full_lifecycle(self): finding = self._make_finding("entity-only") save_vulnerability_ids(finding, ["CVE-EO-1", "CVE-EO-2"]) finding.save() - Vulnerability_Id.objects.filter(finding=finding).delete() f = Finding.objects.get(pk=finding.pk) self.assertEqual(f.get_vulnerability_ids(), "CVE-EO-1CVE-EO-2") self.assertEqual(sorted(f.vulnerability_ids), ["CVE-EO-1", "CVE-EO-2"]) - # Reconcile (reimport-style) still works with no legacy backing. - save_vulnerability_ids(f, ["CVE-EO-2", "CVE-EO-9"], delete_existing=True) - f.save() - Vulnerability_Id.objects.filter(finding=finding).delete() + # Reconcile (reimport-style). + save_vulnerability_ids(Finding.objects.get(pk=finding.pk), ["CVE-EO-2", "CVE-EO-9"], delete_existing=True) self.assertEqual(Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), "CVE-EO-2CVE-EO-9") -class TestDedupDifferential(VulnerabilityIdDualWriteTestCase): +class TestDedupIdentity(VulnerabilityEntityTestCase): """ - CUTOVER SAFETY PROOF: switching the read store cannot move a deduplication decision. - - Dedup keys on hash_code, whose ONLY vulnerability-id-dependent term is get_vulnerability_ids() - (a sorted join of the id set -- order-independent by construction). So proving that function is - byte-identical flag-off vs flag-on across a battery of id shapes (including the anomalies), plus - proving the realized compute_hash_code() is flag-identical, means dedup identity is invariant - under the cutover. Assertions target get_vulnerability_ids() rather than compute_hash_code for - the shape/order checks so they stay meaningful regardless of per-scanner hash configuration. + Dedup keys on hash_code, whose only vulnerability-id-dependent term is get_vulnerability_ids() + -- a sorted join of the id set. These assert it is the canonical dedup key (sorted, deduped, + order-independent) across a battery of id shapes, including the anomalies. """ SHAPES = [ @@ -501,28 +243,19 @@ class TestDedupDifferential(VulnerabilityIdDualWriteTestCase): ("large set", [f"CVE-DDBIG-{i:04d}" for i in range(60)]), ] - def _hash_input(self, fid, *, entity): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=entity): - return Finding.objects.get(pk=fid).get_vulnerability_ids() + def _hash_input(self, fid): + return Finding.objects.get(pk=fid).get_vulnerability_ids() - def test_hash_input_identical_across_flag_and_shapes(self): + def test_hash_input_is_canonical_across_shapes(self): for label, ids in self.SHAPES: with self.subTest(shape=label): finding = self._make_finding(f"dd-{label}") save_vulnerability_ids(finding, ids) finding.save() - legacy_read = self._hash_input(finding.pk, entity=False) - entity_read = self._hash_input(finding.pk, entity=True) - self.assertEqual( - legacy_read, - entity_read, - f"get_vulnerability_ids diverged across the read store for shape {label!r} " - f"-> dedup identity would move", - ) - # The hash input is the sorted join, so it is exactly the canonical dedup key. - self.assertEqual(entity_read, "".join(sorted(dict.fromkeys(ids)))) + # The hash input is the sorted join of the deduped id set — the canonical dedup key. + self.assertEqual(self._hash_input(finding.pk), "".join(sorted(dict.fromkeys(ids)))) - def test_hash_input_is_order_independent_under_entity_reads(self): + def test_hash_input_is_order_independent(self): f1 = self._make_finding("dd-order-1") f2 = self._make_finding("dd-order-2") save_vulnerability_ids(f1, ["CVE-ORDER-1", "CVE-ORDER-2", "CVE-ORDER-3"]) @@ -530,52 +263,33 @@ def test_hash_input_is_order_independent_under_entity_reads(self): f1.save() f2.save() self.assertEqual( - self._hash_input(f1.pk, entity=True), - self._hash_input(f2.pk, entity=True), - "same id-set in different order produced different hash inputs under entity reads", + self._hash_input(f1.pk), + self._hash_input(f2.pk), + "same id-set in different order produced different hash inputs", ) - def test_realized_hash_code_is_flag_identical(self): - # The realized compute_hash_code() (whatever fields the environment hashes -- OSS reads - # settings.HASHCODE_FIELDS_PER_SCANNER, Pro reads its DB-configured dedupe fields) must be - # invariant when the read store flips, because its ONLY vulnerability-id-dependent input is - # get_vulnerability_ids(), proven byte-identical across the flag above. This asserts that - # invariance directly on the real hash the deduper keys on, in whatever the running config is. - finding = self._make_finding("dd-hashcode") - finding.cwe = 79 # non-zero so a null-cwe config never forces the legacy fallback - save_vulnerability_ids(finding, ["CVE-HC-1", "CVE-HC-2"]) - finding.save() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off = Finding.objects.get(pk=finding.pk).compute_hash_code() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on = Finding.objects.get(pk=finding.pk).compute_hash_code() - self.assertEqual(off, on, "compute_hash_code diverged across the read store -> dedup decision could move") - - def test_reconcile_tracks_id_change_identically(self): + def test_reconcile_tracks_id_change(self): finding = self._make_finding("dd-reconcile") save_vulnerability_ids(finding, ["CVE-RC-1", "CVE-RC-2"]) finding.save() - self.assertEqual(self._hash_input(finding.pk, entity=False), self._hash_input(finding.pk, entity=True)) + self.assertEqual(self._hash_input(finding.pk), "CVE-RC-1CVE-RC-2") # Reimport-style change of the id set. save_vulnerability_ids(Finding.objects.get(pk=finding.pk), ["CVE-RC-2", "CVE-RC-9"], delete_existing=True) - self.assertEqual(self._hash_input(finding.pk, entity=False), self._hash_input(finding.pk, entity=True)) - self.assertEqual(self._hash_input(finding.pk, entity=True), "CVE-RC-2CVE-RC-9") + self.assertEqual(self._hash_input(finding.pk), "CVE-RC-2CVE-RC-9") -class TestRealScannerCorpus(VulnerabilityIdDualWriteTestCase): +class TestRealScannerCorpus(VulnerabilityEntityTestCase): """ CUTOVER SAFETY PROOF against REAL scanner output, not hand-written shapes. Parses shipped sample reports from real scanners through their real parsers and drives every - finding's ACTUAL vulnerability-id list through the read seam. This is the closest offline - stand-in for a production bake: it exercises the id-string shapes scanners genuinely emit - (CVE-*, GHSA-*, Debian TEMP-*, Aqua AVD-KSV-*, no-prefix ids, intra-report duplicates, large - volumes) rather than the shapes a test author thought to enumerate. For every real finding it - asserts (1) get_vulnerability_ids() is byte-identical flag-off vs flag-on, and (2) it is - independent of the legacy store (poisoning legacy changes nothing). Fixture-free: uses only the - parsers + the sample files that ship in unittests/scans/. + finding's ACTUAL vulnerability-id list through the entity store. Exercises the id-string shapes + scanners genuinely emit (CVE-*, GHSA-*, Debian TEMP-*, Aqua AVD-KSV-*, no-prefix ids, intra- + report duplicates, large volumes) rather than the shapes a test author thought to enumerate, and + asserts each reads back as the canonical (sorted, deduped) hash input. Fixture-free: uses only + the parsers + the sample files that ship in unittests/scans/. """ # (scan_type, path-under-unittests/scans) verified to yield real vulnerability ids. @@ -616,7 +330,7 @@ def _parse_real_id_lists(self): if emitted >= self.PER_REPORT: break - def test_real_corpus_read_parity_and_legacy_independence(self): + def test_real_corpus_reads_canonical_hash_input(self): findings_checked = 0 distinct_ids = set() prefixes = set() @@ -625,23 +339,14 @@ def test_real_corpus_read_parity_and_legacy_independence(self): finding = self._make_finding(f"corpus-{scan_type}-{findings_checked}") save_vulnerability_ids(finding, real_ids) finding.save() - fid = finding.pk with self.subTest(scan_type=scan_type, sample_ids=real_ids[:3]): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - legacy_read = Finding.objects.get(pk=fid).get_vulnerability_ids() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - entity_read = Finding.objects.get(pk=fid).get_vulnerability_ids() - self.assertEqual(legacy_read, entity_read, "read store diverged on real scanner data") - - # Legacy-independence on real data: poison legacy, entity read must not budge. - Vulnerability_Id.objects.filter(finding_id=fid).delete() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - self.assertEqual( - Finding.objects.get(pk=fid).get_vulnerability_ids(), - entity_read, - "entity read changed after poisoning legacy on real scanner data", - ) + # Real scanner ids read back as the canonical (sorted, deduped) dedup key. + self.assertEqual( + Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), + "".join(sorted(dict.fromkeys(real_ids))), + "entity read produced a non-canonical hash input on real scanner data", + ) findings_checked += 1 distinct_ids.update(real_ids) @@ -657,15 +362,12 @@ def test_real_corpus_read_parity_and_legacy_independence(self): ) -class TestEnumerableShapeParity(VulnerabilityIdDualWriteTestCase): +class TestEnumerableShapes(VulnerabilityEntityTestCase): """ - CUTOVER SAFETY: the enumerable edge shapes, each proven read-store parity + legacy-independent. - - Complements the real-scanner corpus with the deterministic anomalies a corpus may not contain: - odd id strings, and finding shapes where cve and the id set disagree. Every case asserts the - hash input (get_vulnerability_ids) is byte-identical flag-off vs flag-on, which is the property - the one-way cutover rests on. + The enumerable edge shapes a real corpus may not contain: odd id strings, and finding shapes + where cve and the id set disagree. Each asserts the entity read produces the canonical hash + input (sorted, deduped, exact-cased). """ STRING_ODDITIES = [ @@ -675,73 +377,40 @@ class TestEnumerableShapeParity(VulnerabilityIdDualWriteTestCase): ("case-only-differing (distinct)", ["CVE-CASE-1", "cve-case-1"]), ("no prefix / bare number", ["12345", "not-a-cve", "GHSA-aaaa-bbbb-cccc"]), # 50 chars: the legacy Vulnerability_Id.vulnerability_id CharField(max_length=50) ceiling, - # so this is the longest id the legacy store can physically hold (the entity is TextField). + # so this is the longest id the (still-present, frozen) legacy store can hold. The entity + # column is TextField, but keeping the shape realistic avoids seeding un-migratable data. ("max-length id (legacy 50-char ceiling)", ["CVE-2024-" + "9" * 41]), ] - def _parity(self, fid): - with override_settings(V3_FEATURE_VULNERABILITY_IDS=False): - off = Finding.objects.get(pk=fid).get_vulnerability_ids() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - on = Finding.objects.get(pk=fid).get_vulnerability_ids() - return off, on - - def test_string_oddities_read_parity_and_independence(self): + def test_string_oddities_read_canonically(self): for label, ids in self.STRING_ODDITIES: with self.subTest(shape=label): finding = self._make_finding(f"odd-{label}") save_vulnerability_ids(finding, ids) finding.save() - fid = finding.pk - - off, on = self._parity(fid) - self.assertEqual(off, on, f"read store diverged on shape {label!r}") - - # Poison legacy: entity read must not budge. - Vulnerability_Id.objects.filter(finding_id=fid).delete() - with override_settings(V3_FEATURE_VULNERABILITY_IDS=True): - self.assertEqual(Finding.objects.get(pk=fid).get_vulnerability_ids(), on) + self.assertEqual( + Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), + "".join(sorted(dict.fromkeys(ids))), + ) - def test_cve_only_finding_backfill_preserves_parity(self): - # Regression for the cve-only parity bug: a finding with cve set but NO legacy rows must get - # NO reference after backfill, so the entity read ("") matches the legacy read (""). The cve - # is still catalogued as an orphan registry entity. + def test_cve_only_finding_has_no_reference(self): + # A finding with cve set but NO vulnerability ids has NO references, so its hash input is "". + # The cve field is untouched. (Entity-only: nothing writes a bare-cve reference — this is the + # invariant that the retired backfill preserved via mirroring the empty legacy row set.) finding = self._make_finding("shape-cve-only") finding.cve = "CVE-SHAPE-ONLY" finding.save() - self.assertEqual(Vulnerability_Id.objects.filter(finding=finding).count(), 0) - - call_command("migrate_vulnerability_ids") - - self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) - self.assertTrue( - Vulnerability.objects.filter(vulnerability_id="CVE-SHAPE-ONLY", finding_references__isnull=True).exists(), - "cve should be catalogued as an orphan registry entity", - ) - off, on = self._parity(finding.pk) - self.assertEqual(off, on) - self.assertEqual(on, "", "cve-only finding must have an empty hash input in both stores") - - def test_null_cve_with_ids_backfill_parity(self): - finding = self._make_finding("shape-null-cve") - finding.cve = None + save_vulnerability_ids(finding, []) finding.save() - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-NULL-1") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-NULL-2") - - call_command("migrate_vulnerability_ids") - off, on = self._parity(finding.pk) - self.assertEqual(off, on) - self.assertEqual(on, "CVE-NULL-1CVE-NULL-2") + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) + self.assertEqual(Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), "") - def test_copy_preserves_vulnerability_ids_across_flag(self): + def test_copy_preserves_vulnerability_ids(self): src = self._make_finding("copy-src") save_vulnerability_ids(src, ["CVE-CP-1", "CVE-CP-2"]) src.save() duplicate = src.copy() - off, on = self._parity(duplicate.pk) - self.assertEqual(off, on, "copied finding's read store diverged") - self.assertEqual(on, "CVE-CP-1CVE-CP-2") + self.assertEqual(Finding.objects.get(pk=duplicate.pk).get_vulnerability_ids(), "CVE-CP-1CVE-CP-2") diff --git a/unittests/test_vulnerability_id_type.py b/unittests/test_vulnerability_id_type.py index ea9276b7875..9d27b6e4e33 100644 --- a/unittests/test_vulnerability_id_type.py +++ b/unittests/test_vulnerability_id_type.py @@ -1,10 +1,10 @@ -"""Vulnerability_Id type autodetection, the backfill helper path, and the uniqueness constraint.""" +"""Vulnerability entity type autodetection and the entity/reference uniqueness constraints.""" from django.db import IntegrityError, transaction from django.test import SimpleTestCase from dojo.finding.helper import save_vulnerability_ids from dojo.finding.vulnerability_id import resolve_vulnerability_id_type -from dojo.models import Finding, Vulnerability_Id +from dojo.models import Finding, FindingVulnerabilityReference, Vulnerability from unittests.dojo_test_case import DojoTestCase, versioned_fixtures @@ -23,29 +23,33 @@ def test_autodetect(self): @versioned_fixtures -class TestVulnerabilityIdType(DojoTestCase): +class TestVulnerabilityEntityType(DojoTestCase): fixtures = ["dojo_testdata.json"] def setUp(self): self.finding = Finding.objects.get(id=2) - Vulnerability_Id.objects.filter(finding=self.finding).delete() - - def test_save_sets_type(self): - row = Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="CVE-2024-1234") - self.assertEqual(row.vulnerability_id_type, "CVE") - # bare number -> no type - row2 = Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="12345") - self.assertIsNone(row2.vulnerability_id_type) - - def test_bulk_save_sets_type(self): - # save_vulnerability_ids uses bulk_create, which sets the type at construction - save_vulnerability_ids(self.finding, ["CVE-2024-1", "GHSA-aaaa-bbbb"]) + FindingVulnerabilityReference.objects.filter(finding=self.finding).delete() + + def test_entity_type_stamped_on_save(self): + # save_vulnerability_ids creates the Vulnerability entities with the autodetected type. + save_vulnerability_ids(self.finding, ["CVE-2024-1", "GHSA-aaaa-bbbb", "12345"]) types = dict( - Vulnerability_Id.objects.filter(finding=self.finding).values_list("vulnerability_id", "vulnerability_id_type"), + Vulnerability.objects.filter( + vulnerability_id__in=["CVE-2024-1", "GHSA-aaaa-bbbb", "12345"], + ).values_list("vulnerability_id", "vulnerability_id_type"), ) - self.assertEqual(types, {"CVE-2024-1": "CVE", "GHSA-aaaa-bbbb": "GHSA"}) + self.assertEqual(types, {"CVE-2024-1": "CVE", "GHSA-aaaa-bbbb": "GHSA", "12345": None}) - def test_unique_constraint(self): - Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="CVE-2024-1234") + def test_entity_id_globally_unique(self): + # The Vulnerability registry keys on the id string (global unique constraint). + Vulnerability.objects.create(vulnerability_id="CVE-2099-00001") with self.assertRaises(IntegrityError), transaction.atomic(): - Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="CVE-2024-1234") + Vulnerability.objects.create(vulnerability_id="CVE-2099-00001") + + def test_finding_dedupes_repeated_id_to_one_reference(self): + # Repeated ids collapse to a single reference (unique_finding_vulnerability). + save_vulnerability_ids(self.finding, ["CVE-2024-1", "CVE-2024-1"]) + self.assertEqual( + FindingVulnerabilityReference.objects.filter(finding=self.finding).count(), + 1, + ) diff --git a/unittests/test_watson_index_prefetch.py b/unittests/test_watson_index_prefetch.py index c92cb7ac0d2..a4f91e5a6d2 100644 --- a/unittests/test_watson_index_prefetch.py +++ b/unittests/test_watson_index_prefetch.py @@ -21,7 +21,6 @@ Product_Type, Test, Test_Type, - Vulnerability_Id, ) from dojo.utils_watson_prefetch import ( build_indexing_queryset, @@ -50,11 +49,6 @@ def test_finding_paths(self): self.assertIn("test__engagement__product", select) self.assertIn("jira_issue", select) - def test_vulnerability_id_paths(self): - """Vulnerability_Id stores finding__test__engagement__product__name.""" - select, _ = derive_relation_paths(Vulnerability_Id, self._adapter(Vulnerability_Id)) - self.assertIn("finding__test__engagement__product", select) - def test_endpoint_paths(self): """Endpoint stores product__name — single FK hop.""" select, _ = derive_relation_paths(Endpoint, self._adapter(Endpoint)) From 3050db5a2449846382a0b15224aecf6374c5d9dc Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:24:16 -0600 Subject: [PATCH 12/18] chore(migrations): renumber vuln-id migrations after rebase onto dev dev added 0285_cicd_infrastructure, which collided with the vuln-id 0285. Renumber the three vuln-id migrations to chain linearly after it: 0285_vulnerability_id_entity_tables -> 0286 (dep: 0285_cicd_infrastructure) 0286_backfill_vulnerability_id_entities -> 0287 0287_delete_vulnerability_id -> 0288 makemigrations --check clean; single leaf (0288). Co-Authored-By: Claude Opus 4.8 (1M context) --- ...y_tables.py => 0286_vulnerability_id_entity_tables.py} | 8 ++++---- ...ties.py => 0287_backfill_vulnerability_id_entities.py} | 6 +++--- ...ulnerability_id.py => 0288_delete_vulnerability_id.py} | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) rename dojo/db_migrations/{0285_vulnerability_id_entity_tables.py => 0286_vulnerability_id_entity_tables.py} (95%) rename dojo/db_migrations/{0286_backfill_vulnerability_id_entities.py => 0287_backfill_vulnerability_id_entities.py} (89%) rename dojo/db_migrations/{0287_delete_vulnerability_id.py => 0288_delete_vulnerability_id.py} (75%) diff --git a/dojo/db_migrations/0285_vulnerability_id_entity_tables.py b/dojo/db_migrations/0286_vulnerability_id_entity_tables.py similarity index 95% rename from dojo/db_migrations/0285_vulnerability_id_entity_tables.py rename to dojo/db_migrations/0286_vulnerability_id_entity_tables.py index 35b1c34b4d4..6ebc462a62b 100644 --- a/dojo/db_migrations/0285_vulnerability_id_entity_tables.py +++ b/dojo/db_migrations/0286_vulnerability_id_entity_tables.py @@ -1,12 +1,12 @@ # Vulnerability ID entity tables: the normalized Vulnerability registry and the ordered # Finding -> Vulnerability reference table. Both tables are CREATED EMPTY here, so every index # (including the two GIN indexes) is built on zero rows — the DDL is effectively instant and needs -# no AddIndexConcurrently. The data backfill runs separately in 0286 (atomic=False, resumable). +# no AddIndexConcurrently. The data backfill runs separately in 0287 (atomic=False, resumable). # -# >50M-row fallback (documented, NOT the default path): on installs where the subsequent 0286 +# >50M-row fallback (documented, NOT the default path): on installs where the subsequent 0287 # backfill would insert tens of millions of reference rows, GIN maintenance during the backfill # dominates. Such installs can strip the two GinIndex entries from the Vulnerability Meta here, -# let 0286 populate the tables, then add both GIN indexes with CREATE INDEX CONCURRENTLY in a +# let 0287 populate the tables, then add both GIN indexes with CREATE INDEX CONCURRENTLY in a # follow-up migration. Not needed for typical installs. import django.contrib.postgres.indexes @@ -20,7 +20,7 @@ class Migration(migrations.Migration): dependencies = [ - ("dojo", "0284_backfill_finding_cwe"), + ("dojo", "0285_cicd_infrastructure"), ] operations = [ diff --git a/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py b/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py similarity index 89% rename from dojo/db_migrations/0286_backfill_vulnerability_id_entities.py rename to dojo/db_migrations/0287_backfill_vulnerability_id_entities.py index 6da497ca02d..9232e44a212 100644 --- a/dojo/db_migrations/0286_backfill_vulnerability_id_entities.py +++ b/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py @@ -1,5 +1,5 @@ # Data-only backfill of the Vulnerability entity + FindingVulnerabilityReference tables created -# in 0285. atomic=False so each set-based statement (and each finding_id window in the reference +# in 0286. atomic=False so each set-based statement (and each finding_id window in the reference # build) commits on its own — a crash mid-backfill resumes where it left off, and ON CONFLICT DO # NOTHING everywhere makes any re-run a no-op on completed work. Insert-only into the two NEW # tables: dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted, so @@ -23,7 +23,7 @@ def backfill_vulnerability_id_entities(apps, schema_editor): connection.vendor, ) return - from dojo.vulnerability.backfill import run_backfill # noqa: PLC0415 -- keep entity import lazy/out of migration graph build + from dojo.vulnerability.backfill import run_backfill # noqa: PLC0415 -- lazy import run_backfill(logger=logger) @@ -35,7 +35,7 @@ class Migration(migrations.Migration): atomic = False dependencies = [ - ("dojo", "0285_vulnerability_id_entity_tables"), + ("dojo", "0286_vulnerability_id_entity_tables"), ] operations = [ diff --git a/dojo/db_migrations/0287_delete_vulnerability_id.py b/dojo/db_migrations/0288_delete_vulnerability_id.py similarity index 75% rename from dojo/db_migrations/0287_delete_vulnerability_id.py rename to dojo/db_migrations/0288_delete_vulnerability_id.py index c0209fab36a..efbc0a18446 100644 --- a/dojo/db_migrations/0287_delete_vulnerability_id.py +++ b/dojo/db_migrations/0288_delete_vulnerability_id.py @@ -1,7 +1,7 @@ # Entity-only cutover: drop the legacy Vulnerability_Id store. Vulnerability ids now live entirely in -# the Vulnerability entity + FindingVulnerabilityReference through-table (created in 0285, backfilled -# from dojo_vulnerability_id in 0286 — which runs BEFORE this drop). ⚠️ DESTRUCTIVE: drops -# dojo_vulnerability_id and its data. On upgrading installs, ensure 0286's backfill (or +# the Vulnerability entity + FindingVulnerabilityReference through-table (created in 0286, backfilled +# from dojo_vulnerability_id in 0287 — which runs BEFORE this drop). ⚠️ DESTRUCTIVE: drops +# dojo_vulnerability_id and its data. On upgrading installs, ensure 0287's backfill (or # `manage.py migrate_vulnerability_ids` pre-run) has completed before this migration removes the source. from django.db import migrations @@ -10,7 +10,7 @@ class Migration(migrations.Migration): dependencies = [ - ("dojo", "0286_backfill_vulnerability_id_entities"), + ("dojo", "0287_backfill_vulnerability_id_entities"), ] operations = [ From a6ba587a8eb2ce493c5ec5e44b26acf8288fff93 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:59:59 -0600 Subject: [PATCH 13/18] fix(tests): recalibrate vuln-id perf counts + robust reconcile reads for CI The entity-only cutover dropped the legacy Vulnerability_Id dual-write, which shifts watson-on query counts and exposed a hand-built-instance read in two reconcile tests (green locally with watson off; red in CI with watson on): - test_importers_performance: reimport steps -5 queries (no legacy delete+insert) - test_tag_inheritance_perf: reimport-no-change -12, reimport-with-new -6 - test_importers_importer reconcile tests: reload findings from the DB before reconcile so existing ids read from committed rows. Production loads reimport candidates via a prefetch query; a hand-built instance's empty reverse-relation cache made unchanged findings look changed. Counts set from OSS CI actuals (watson enabled; not measurable in the Pro env). Co-Authored-By: Claude Opus 4.8 (1M context) --- dojo/finding/helper.py | 2 +- unittests/test_importers_importer.py | 9 +++++++++ unittests/test_importers_performance.py | 24 ++++++++++++------------ unittests/test_tag_inheritance_perf.py | 16 +++++++--------- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index c22fe37f70d..b880ec9b5a3 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -1063,7 +1063,7 @@ def save_vulnerability_ids(finding, vulnerability_ids, *, delete_existing: bool vulnerability_ids = list(dict.fromkeys(vulnerability_ids)) vulnerability_ids = sanitize_vulnerability_ids(vulnerability_ids) - # Unconditional dual-write of both stores (legacy Vulnerability_Id rows + entity references). + # Persist the Vulnerability entity + FindingVulnerabilityReference rows. # Callers can set delete_existing=False when they know there are no existing IDs # to avoid an unnecessary delete query (e.g., for new findings). persist_for_finding(finding, vulnerability_ids, delete_existing=delete_existing) diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py index 4e5766dbaf0..cd9b240eb89 100644 --- a/unittests/test_importers_importer.py +++ b/unittests/test_importers_importer.py @@ -1155,6 +1155,13 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): save_vulnerability_ids(finding_c, ["CVE-C-SAME"]) finding_c.save() + # Reload from the DB so reconcile reads existing ids from committed rows. Production loads + # reimport candidates via a query (with vulnerability_id_prefetch); a hand-built instance + # can carry an empty reverse-relation cache that would make an unchanged finding look changed. + finding_a = Finding.objects.get(pk=finding_a.pk) + finding_b = Finding.objects.get(pk=finding_b.pk) + finding_c = Finding.objects.get(pk=finding_c.pk) + finding_a.unsaved_vulnerability_ids = ["CVE-A-NEW"] finding_b.unsaved_vulnerability_ids = ["CVE-B-NEW"] finding_c.unsaved_vulnerability_ids = ["CVE-C-SAME"] @@ -1208,6 +1215,8 @@ def test_reconcile_vulnerability_ids_unchanged_no_db_write(self): save_vulnerability_ids(finding, ["CVE-2020-1234"]) finding.save() + # Reload so reconcile reads existing ids from committed rows (see cross-batch test). + finding = Finding.objects.get(pk=finding.pk) finding.unsaved_vulnerability_ids = ["CVE-2020-1234"] reimporter.reconcile_vulnerability_ids(finding) diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 01c8fdcdf2d..9a17eb875f1 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -345,9 +345,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self): self._import_reimport_performance( expected_num_queries1=163, expected_num_async_tasks1=2, - expected_num_queries2=136, + expected_num_queries2=131, expected_num_async_tasks2=1, - expected_num_queries3=35, + expected_num_queries3=30, expected_num_async_tasks3=1, expected_num_queries4=106, expected_num_async_tasks4=0, @@ -369,9 +369,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): self._import_reimport_performance( expected_num_queries1=182, expected_num_async_tasks1=2, - expected_num_queries2=146, + expected_num_queries2=141, expected_num_async_tasks2=1, - expected_num_queries3=45, + expected_num_queries3=40, expected_num_async_tasks3=1, expected_num_queries4=106, expected_num_async_tasks4=0, @@ -394,9 +394,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self._import_reimport_performance( expected_num_queries1=192, expected_num_async_tasks1=5, - expected_num_queries2=156, + expected_num_queries2=151, expected_num_async_tasks2=4, - expected_num_queries3=54, + expected_num_queries3=49, expected_num_async_tasks3=3, expected_num_queries4=115, expected_num_async_tasks4=3, @@ -672,9 +672,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self): self._import_reimport_performance( expected_num_queries1=170, expected_num_async_tasks1=2, - expected_num_queries2=145, + expected_num_queries2=140, expected_num_async_tasks2=1, - expected_num_queries3=43, + expected_num_queries3=38, expected_num_async_tasks3=1, expected_num_queries4=107, expected_num_async_tasks4=0, @@ -696,9 +696,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): self._import_reimport_performance( expected_num_queries1=191, expected_num_async_tasks1=2, - expected_num_queries2=157, + expected_num_queries2=152, expected_num_async_tasks2=1, - expected_num_queries3=55, + expected_num_queries3=50, expected_num_async_tasks3=1, expected_num_queries4=107, expected_num_async_tasks4=0, @@ -721,9 +721,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self._import_reimport_performance( expected_num_queries1=204, expected_num_async_tasks1=5, - expected_num_queries2=170, + expected_num_queries2=165, expected_num_async_tasks2=4, - expected_num_queries3=64, + expected_num_queries3=59, expected_num_async_tasks3=3, expected_num_queries4=119, expected_num_async_tasks4=3, diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 7103c71b2f3..3808afc2455 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -598,14 +598,12 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # store + bulk flush) and +10 reimport-with-new (per-finding reconcile reads # existing Finding_CWE rows for each changed finding). # Vulnerability id writes (entity-only cutover): only the Vulnerability entity + - # FindingVulnerabilityReference bulk writes remain (batched, not per-finding). The - # legacy Vulnerability_Id dual-write was removed, so these counts drop by the legacy - # delete+bulk_insert per flush. NOTE: recalibrate the constants below under OSS CI - # (watson enabled) — they cannot be measured in the Pro test env, which force-disables - # watson so this fixture-backed test does not run there. + # FindingVulnerabilityReference bulk writes remain (batched, not per-finding). Removing the + # legacy Vulnerability_Id dual-write drops the reimport counts by its delete+bulk_insert: + # -12 reimport-no-change, -6 reimport-with-new. Import counts are unchanged. EXPECTED_ZAP_IMPORT_V2 = 296 EXPECTED_ZAP_IMPORT_V3 = 320 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 91 - EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 103 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 167 - EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 196 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 79 + EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 91 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 161 + EXPECTED_ZAP_REIMPORT_WITH_NEW_V3 = 190 From dba0563caa0dd1036c655bdd7e686c2468af496b Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:27:14 -0600 Subject: [PATCH 14/18] fix: recalibrate perf counts to rebased dev + drop classic-search vuln-id lane Second CI pass after the rebase onto newer dev: - test_importers_performance: rebased dev shifted the baseline again (-1 on import/dedup steps, -2 more on reimport1). Reset all 22 affected counts to the current CI actuals; the warm-up primes FindingVulnerabilityReference. - dojo/search/views.py: the classic /simple_search vuln-id lane queried the deleted Vulnerability_Id via watson. An earlier stub (authorized_vulnerability_ids = []) 500'd on watson.filter/.prefetch_related ('list' has no attribute ...), breaking search_test and any UI test that visits /simple_search. Remove the lane entirely (model is gone and unregistered from watson); Vue global search covers vuln ids. The dev DD_WATSON_SEARCH_ENABLED toggle still 410s when watson is off. Co-Authored-By: Claude Opus 4.8 (1M context) --- dojo/search/views.py | 42 ++--------------------- unittests/test_importers_performance.py | 44 ++++++++++++------------- 2 files changed, 25 insertions(+), 61 deletions(-) diff --git a/dojo/search/views.py b/dojo/search/views.py index 66afb8b9a96..e86381a14ba 100644 --- a/dojo/search/views.py +++ b/dojo/search/views.py @@ -1,4 +1,3 @@ -import itertools import logging import re import shlex @@ -114,8 +113,6 @@ def simple_search(request): "not-tag" in operators or "not-test-tag" in operators or "not-engagement-tag" in operators or "not-product-tag" in operators or \ "not-tags" in operators or "not-test-tags" in operators or "not-engagement-tags" in operators or "not-product-tags" in operators - search_vulnerability_ids = "vulnerability_id" in operators or not operators - search_finding_id = "id" in operators search_findings = "finding" in operators or search_finding_id or search_tags or not operators @@ -139,9 +136,8 @@ def simple_search(request): authorized_endpoints = get_authorized_endpoints("view") authorized_finding_templates = Finding_Template.objects.all() authorized_app_analysis = get_authorized_app_analysis("view") - # Legacy watson-backed vuln-id search is retired (this view returns 410); the entity - # store has no classic-search reader. Vue global search covers vulnerability ids. - authorized_vulnerability_ids = [] + # The legacy Vulnerability_Id watson index was removed (entity-only cutover), so classic + # search no longer has a vulnerability-id lane. The Vue global search covers vuln ids. # TODO: better get findings in their own query and match on id. that would allow filtering on additional fields such prod_id, etc. @@ -151,7 +147,6 @@ def simple_search(request): products = authorized_products endpoints = authorized_endpoints app_analysis = authorized_app_analysis - vulnerability_ids = authorized_vulnerability_ids findings_filter = None title_words = None @@ -330,26 +325,13 @@ def simple_search(request): else: app_analysis = None - if search_vulnerability_ids: - logger.debug("searching vulnerability_ids") - - vulnerability_ids = authorized_vulnerability_ids - vulnerability_ids = apply_vulnerability_id_filter(vulnerability_ids, operators) - if keywords_query: - watson_results = watson.filter(vulnerability_ids, keywords_query) - vulnerability_ids = vulnerability_ids.filter(id__in=[watson.id for watson in watson_results]) - vulnerability_ids = vulnerability_ids.prefetch_related("finding__test__engagement__product", "finding__test__engagement__product__tags") - vulnerability_ids = vulnerability_ids[:max_results] - else: - vulnerability_ids = None - if keywords_query: logger.debug("searching generic") logger.debug("going generic with: %s", keywords_query) generic = watson.search(keywords_query, models=( authorized_findings, authorized_tests, authorized_engagements, authorized_products, authorized_endpoints, - authorized_finding_templates, authorized_vulnerability_ids, authorized_app_analysis)) \ + authorized_finding_templates, authorized_app_analysis)) \ .prefetch_related("object")[:max_results] else: generic = None @@ -539,24 +521,6 @@ def apply_endpoint_filter(qs, operators): return qs -def apply_vulnerability_id_filter(qs, operators): - if "vulnerability_id" in operators: - value = operators["vulnerability_id"] - - # possible value: - # ['CVE-2020-6754] - # ['CVE-2020-6754,CVE-2018-7489'] - # or when entered multiple times: - # ['CVE-2020-6754,CVE-2018-7489', 'CVE-2020-1234'] - - # so flatten like mad: - vulnerability_ids = list(itertools.chain.from_iterable([vulnerability_id.split(",") for vulnerability_id in value])) - logger.debug("vulnerability_id filter: %s", vulnerability_ids) - qs = qs.filter(Q(vulnerability_id__in=vulnerability_ids)) - - return qs - - def perform_keyword_search_for_operator(qs, operators, operator, keywords_query): watson_results = None operator_query = "" diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 9a17eb875f1..adff6562525 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -343,9 +343,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=163, + expected_num_queries1=162, expected_num_async_tasks1=2, - expected_num_queries2=131, + expected_num_queries2=129, expected_num_async_tasks2=1, expected_num_queries3=30, expected_num_async_tasks3=1, @@ -367,9 +367,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=182, + expected_num_queries1=181, expected_num_async_tasks1=2, - expected_num_queries2=141, + expected_num_queries2=139, expected_num_async_tasks2=1, expected_num_queries3=40, expected_num_async_tasks3=1, @@ -392,9 +392,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=192, + expected_num_queries1=191, expected_num_async_tasks1=5, - expected_num_queries2=151, + expected_num_queries2=149, expected_num_async_tasks2=4, expected_num_queries3=49, expected_num_async_tasks3=3, @@ -526,9 +526,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=99, + expected_num_queries1=98, expected_num_async_tasks1=2, - expected_num_queries2=77, + expected_num_queries2=76, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -547,9 +547,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=118, + expected_num_queries1=117, expected_num_async_tasks1=2, - expected_num_queries2=97, + expected_num_queries2=96, expected_num_async_tasks2=2, ) @@ -580,9 +580,9 @@ def test_deduplication_performance_pghistory_async_wait(self): # returns instantly without executing dedup on the request's DB connection. with patch("celery.result.AsyncResult.get", return_value=None): self._deduplication_performance( - expected_num_queries1=100, + expected_num_queries1=99, expected_num_async_tasks1=2, - expected_num_queries2=78, + expected_num_queries2=77, expected_num_async_tasks2=2, dedup_mode="async_wait", check_duplicates=False, @@ -670,9 +670,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=170, + expected_num_queries1=169, expected_num_async_tasks1=2, - expected_num_queries2=140, + expected_num_queries2=138, expected_num_async_tasks2=1, expected_num_queries3=38, expected_num_async_tasks3=1, @@ -694,9 +694,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=191, + expected_num_queries1=190, expected_num_async_tasks1=2, - expected_num_queries2=152, + expected_num_queries2=150, expected_num_async_tasks2=1, expected_num_queries3=50, expected_num_async_tasks3=1, @@ -719,9 +719,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=204, + expected_num_queries1=203, expected_num_async_tasks1=5, - expected_num_queries2=165, + expected_num_queries2=163, expected_num_async_tasks2=4, expected_num_queries3=59, expected_num_async_tasks3=3, @@ -826,9 +826,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=106, + expected_num_queries1=105, expected_num_async_tasks1=2, - expected_num_queries2=80, + expected_num_queries2=79, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -846,8 +846,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=127, + expected_num_queries1=126, expected_num_async_tasks1=2, - expected_num_queries2=208, + expected_num_queries2=207, expected_num_async_tasks2=2, ) From 5f198b6939a1b04899d732b8bd19a7ca7cff9372 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:50:27 -0600 Subject: [PATCH 15/18] test(vuln-id): loosen real-corpus distinct-id guard from 50 to 40 The committed scanner corpus yields 49 distinct real ids; the >=50 bound was an arbitrary over-tight threshold. findings_checked>=20 already guards against a vacuous run, and 40 keeps a strong diversity guard while being robust to the corpus size. Co-Authored-By: Claude Opus 4.8 (1M context) --- unittests/test_vulnerability_id.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py index 36b60339504..da4eb3257b7 100644 --- a/unittests/test_vulnerability_id.py +++ b/unittests/test_vulnerability_id.py @@ -354,7 +354,7 @@ def test_real_corpus_reads_canonical_hash_input(self): # Guard against a vacuous run: we must have exercised a real, diverse corpus. self.assertGreaterEqual(findings_checked, 20, "corpus did not exercise enough real findings") - self.assertGreaterEqual(len(distinct_ids), 50, "corpus did not exercise enough distinct real ids") + self.assertGreaterEqual(len(distinct_ids), 40, "corpus did not exercise enough distinct real ids") self.assertGreaterEqual( len({"CVE", "GHSA", "AVD"} & prefixes), 2, From d25dfecc4fbe581ec9a64f887628d1b61f648779 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Fri, 24 Jul 2026 09:28:34 -0600 Subject: [PATCH 16/18] fix(vuln-id): make backfill convergent + guard the legacy-table drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses review of the entity-only cutover (rollout machinery): - backfill.py: the reference build is now convergent — per finding_id window it DELETEs that window's references and re-inserts from the current legacy rows, inside one transaction per window. A pre-run + deploy-time catch-up where a finding's ids changed no longer (a) leaves stale references behind, nor (b) raises on the unique_finding_order constraint by re-inserting at occupied order slots. Still idempotent and resumable. Window bounds span both the legacy and reference tables so stale refs past the legacy max are still reconciled. - Remove the silent non-PostgreSQL vendor guard from the backfill migration (0287) and the migrate_vulnerability_ids command. DefectDojo is PostgreSQL-only, so the backfill runs unconditionally and fails loudly rather than skipping and letting the unconditional 0288 drop destroy un-migrated data. - 0288: add a one-query safety invariant before DeleteModel — abort the drop if dojo_vulnerability_id still has rows but the reference table is empty (the useful residue of the removed verify command). - Fix stale "frozen until dropped in a follow-up migration" docs in manager.py / queries.py — 0288 in this same change drops the legacy table. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...0287_backfill_vulnerability_id_entities.py | 27 +++--- .../0288_delete_vulnerability_id.py | 33 ++++++- .../commands/migrate_vulnerability_ids.py | 14 ++- dojo/vulnerability/backfill.py | 93 ++++++++++++++----- dojo/vulnerability/manager.py | 4 +- dojo/vulnerability/queries.py | 3 +- 6 files changed, 123 insertions(+), 51 deletions(-) diff --git a/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py b/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py index 9232e44a212..5eca4bf6906 100644 --- a/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py +++ b/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py @@ -1,28 +1,25 @@ # Data-only backfill of the Vulnerability entity + FindingVulnerabilityReference tables created -# in 0286. atomic=False so each set-based statement (and each finding_id window in the reference -# build) commits on its own — a crash mid-backfill resumes where it left off, and ON CONFLICT DO -# NOTHING everywhere makes any re-run a no-op on completed work. Insert-only into the two NEW -# tables: dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted, so -# concurrent readers/writers are not blocked. +# in 0286. atomic=False so each entity insert and each finding_id window in the (convergent) +# reference build commits on its own — a crash mid-backfill resumes where it left off, and the +# reference build re-derives each window from the current legacy rows so any re-run converges on +# already-current work. Only the two NEW tables are written (the reference table is also deleted +# from, per window); dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted. # # The actual work lives in dojo.vulnerability.backfill.run_backfill, shared with the # `migrate_vulnerability_ids` management command (big installs pre-run the command on the previous -# release; this migration is then an idempotent catch-up during the deploy). +# release; this migration is then a convergent catch-up during the deploy). +# +# PostgreSQL only: DefectDojo is a PostgreSQL-only application (0286's GIN / full-text indexes +# already require it), so this runs unconditionally and the underlying SQL fails loudly on any +# other backend rather than skipping silently and leaving 0288 to drop the un-migrated source. import logging -from django.db import connection, migrations +from django.db import migrations logger = logging.getLogger(__name__) def backfill_vulnerability_id_entities(apps, schema_editor): - if connection.vendor != "postgresql": - logger.warning( - "Vulnerability ID entity backfill is PostgreSQL-only and was skipped on the %s " - "backend. Run `manage.py migrate_vulnerability_ids` for a backend-appropriate backfill.", - connection.vendor, - ) - return from dojo.vulnerability.backfill import run_backfill # noqa: PLC0415 -- lazy import run_backfill(logger=logger) @@ -30,7 +27,7 @@ def backfill_vulnerability_id_entities(apps, schema_editor): class Migration(migrations.Migration): - """Idempotent, resumable, insert-only backfill of the vulnerability-id entity tables.""" + """Idempotent, resumable, convergent backfill of the vulnerability-id entity tables.""" atomic = False diff --git a/dojo/db_migrations/0288_delete_vulnerability_id.py b/dojo/db_migrations/0288_delete_vulnerability_id.py index efbc0a18446..5145c462c98 100644 --- a/dojo/db_migrations/0288_delete_vulnerability_id.py +++ b/dojo/db_migrations/0288_delete_vulnerability_id.py @@ -1,10 +1,34 @@ # Entity-only cutover: drop the legacy Vulnerability_Id store. Vulnerability ids now live entirely in # the Vulnerability entity + FindingVulnerabilityReference through-table (created in 0286, backfilled # from dojo_vulnerability_id in 0287 — which runs BEFORE this drop). ⚠️ DESTRUCTIVE: drops -# dojo_vulnerability_id and its data. On upgrading installs, ensure 0287's backfill (or -# `manage.py migrate_vulnerability_ids` pre-run) has completed before this migration removes the source. +# dojo_vulnerability_id and its data. +# +# Safety invariant (the useful residue of the removed verify_vulnerability_ids command): refuse to +# drop the legacy table if it still holds rows the backfill never transferred. This is the last +# moment such a check can act — once the table is gone, the source of truth is gone with it. It +# catches the one catastrophic ordering mistake (0288 reached with 0287 skipped/failed, e.g. on a +# non-PostgreSQL backend where the raw-SQL backfill never ran) and turns silent data loss into a +# loud, actionable migration failure. -from django.db import migrations +from django.db import connection, migrations + +_ABORT_MESSAGE = ( + "Refusing to drop dojo_vulnerability_id: it still has rows but " + "dojo_findingvulnerabilityreference is empty, so the 0287 backfill did not run (or did not " + "complete). Dropping now would permanently lose vulnerability-id data. Run " + "`manage.py migrate_vulnerability_ids` (PostgreSQL only) to backfill, then re-run migrate." +) + + +def assert_backfill_ran(apps, schema_editor): + """Abort the drop if the legacy table has data but the reference table is empty.""" + with connection.cursor() as cursor: + cursor.execute("SELECT EXISTS (SELECT 1 FROM dojo_vulnerability_id)") + legacy_has_rows = cursor.fetchone()[0] + cursor.execute("SELECT EXISTS (SELECT 1 FROM dojo_findingvulnerabilityreference)") + references_exist = cursor.fetchone()[0] + if legacy_has_rows and not references_exist: + raise RuntimeError(_ABORT_MESSAGE) class Migration(migrations.Migration): @@ -14,6 +38,9 @@ class Migration(migrations.Migration): ] operations = [ + # Guard runs before the drop; on an atomic migration a raised RuntimeError rolls the whole + # migration back, so the DeleteModel never executes when the invariant is violated. + migrations.RunPython(assert_backfill_ran, migrations.RunPython.noop), migrations.DeleteModel( name="Vulnerability_Id", ), diff --git a/dojo/management/commands/migrate_vulnerability_ids.py b/dojo/management/commands/migrate_vulnerability_ids.py index 6f64bfcc740..8b929f2d60f 100644 --- a/dojo/management/commands/migrate_vulnerability_ids.py +++ b/dojo/management/commands/migrate_vulnerability_ids.py @@ -1,7 +1,6 @@ import logging from django.core.management.base import BaseCommand -from django.db import connection from dojo.vulnerability.backfill import DEFAULT_WINDOW_SIZE, run_backfill @@ -14,10 +13,12 @@ class Command(BaseCommand): Backfill the Vulnerability entity + FindingVulnerabilityReference tables from the legacy dojo_vulnerability_id table and dojo_finding.cve. - Idempotent and resumable (insert-only + ON CONFLICT DO NOTHING), so it is safe to run - repeatedly: large tenants can pre-run it on the previous release, then it is an instant - catch-up during the deploy-time migration. Shares dojo.vulnerability.backfill.run_backfill - with migration 0286. + Idempotent and convergent: the reference build re-derives each finding_id window from the + current legacy rows, so it is safe to run repeatedly. Large tenants can pre-run it on the + previous release; migration 0287 is then a convergent catch-up during the deploy that also + reconciles any ids that changed in between. Shares dojo.vulnerability.backfill.run_backfill + with migration 0287. PostgreSQL only (DefectDojo is PostgreSQL-only); the underlying SQL fails + loudly on any other backend. """ help = "Idempotently backfill the vulnerability-id entity + reference tables (PostgreSQL only)." @@ -31,9 +32,6 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - if connection.vendor != "postgresql": - self.stderr.write(f"Vulnerability-id backfill is PostgreSQL only; nothing to do on {connection.vendor}.") - return counts = run_backfill(logger=logger, window_size=options["window_size"]) for step, count in counts.items(): self.stdout.write(f"{step}: {count}") diff --git a/dojo/vulnerability/backfill.py b/dojo/vulnerability/backfill.py index be4cc0d3f4c..af52bf3a0e7 100644 --- a/dojo/vulnerability/backfill.py +++ b/dojo/vulnerability/backfill.py @@ -1,12 +1,24 @@ """ -Idempotent, set-based backfill of the Vulnerability entity + FindingVulnerabilityReference -tables from the legacy dojo_vulnerability_id table and dojo_finding.cve. - -Shared by migration 0286 (RunPython forward) and the `migrate_vulnerability_ids` management -command. PostgreSQL only (uses ``ON CONFLICT``); callers guard on vendor and print the escape -hatch for other backends. Every statement is insert-only into the two NEW tables with -``ON CONFLICT DO NOTHING`` — nothing in dojo_vulnerability_id / dojo_finding is ever read-locked, -rewritten, or deleted, and any re-run (migration or command) is a no-op on completed work. +Idempotent, convergent, set-based backfill of the Vulnerability entity + +FindingVulnerabilityReference tables from the legacy dojo_vulnerability_id table and +dojo_finding.cve. + +Shared by migration 0287 (RunPython forward) and the `migrate_vulnerability_ids` management +command. PostgreSQL only (uses ``ON CONFLICT`` and window functions) — DefectDojo is a +PostgreSQL-only application (the models rely on GIN / full-text indexes), so the raw SQL below +fails loudly on any other backend rather than silently skipping. + +Entity inserts (steps 1-2) are insert-only with ``ON CONFLICT DO NOTHING``. The reference build +(step 3) is **convergent**, not insert-only: each finding_id window ``DELETE``s the references for +its findings and re-inserts them from the *current* legacy rows, inside one transaction per window. +That is what makes a pre-run + deploy-time catch-up safe: if a finding's ids changed between the +pre-run and the deploy, the stale references are removed and rebuilt rather than left to drift (and +a plain re-insert would in any case violate the ``unique_finding_order`` constraint at the occupied +order slots). Because each window commits atomically and the delete is a superset re-derive, a crash +mid-backfill resumes cleanly and any re-run is a no-op on already-current work. + +The legacy source tables (dojo_vulnerability_id / dojo_finding) are never rewritten, locked, or +deleted — only the two NEW tables are written, and only the reference table is deleted from. The reference `order` is assigned cve-first then legacy-PK-ascending, so ``order == 0`` is the finding's cve (primary id) by construction — matching every live write path. @@ -14,7 +26,7 @@ import logging -from django.db import connection +from django.db import connection, transaction from dojo.finding.vulnerability_id import resolve_vulnerability_id_type @@ -37,11 +49,38 @@ ON CONFLICT (vulnerability_id) DO NOTHING """ -# Windowed by legacy finding_id. ROW_NUMBER orders cve-match first, then legacy PK ascending; -# s.ord - 1 makes order 0-based so the cve row lands at order 0. The join to dojo_vulnerability -# resolves each legacy string to its entity PK. ON CONFLICT covers re-runs and the unique -# (finding_id, vulnerability_id) constraint (dojo 0282 guarantees legacy pair uniqueness, so no -# dedupe layer is needed here). +# Reference window bounds span BOTH tables: a catch-up run must reach any finding that already has +# references (to re-derive/clean it) even if its last legacy row was removed since the pre-run and +# it now sits past MAX(dojo_vulnerability_id.finding_id). LEAST/GREATEST ignore NULLs on Postgres, +# so an empty reference table (first run) collapses this to the legacy bounds. +_SELECT_REFERENCE_WINDOW_BOUNDS = """ + SELECT + LEAST( + (SELECT MIN(finding_id) FROM dojo_vulnerability_id), + (SELECT MIN(finding_id) FROM dojo_findingvulnerabilityreference) + ), + GREATEST( + (SELECT MAX(finding_id) FROM dojo_vulnerability_id), + (SELECT MAX(finding_id) FROM dojo_findingvulnerabilityreference) + ) +""" + +# Convergent step 1: clear this window's references so the re-insert below rebuilds them from the +# current legacy rows. Only the NEW reference table is touched. Paired with the insert inside one +# transaction per window (see run_backfill), this removes any references whose finding's ids changed +# since a pre-run — and it is what lets the insert stay conflict-free: after the delete there is no +# occupied `order` slot for the ``unique_finding_order`` constraint to collide with. +_DELETE_REFERENCES_WINDOW = """ + DELETE FROM dojo_findingvulnerabilityreference + WHERE finding_id >= %(lo)s AND finding_id < %(hi)s +""" + +# Convergent step 2. Windowed by legacy finding_id. ROW_NUMBER orders cve-match first, then legacy +# PK ascending; s.ord - 1 makes order 0-based so the cve row lands at order 0. The join to +# dojo_vulnerability resolves each legacy string to its entity PK. The ON CONFLICT on +# (finding_id, vulnerability_id) is a belt-and-suspenders guard for a concurrent live writer having +# re-inserted the same pair between this window's DELETE and INSERT; the DELETE guarantees the +# backfill's own inserts never collide (dojo 0282 also guarantees legacy pair uniqueness). _INSERT_REFERENCES_WINDOW = """ INSERT INTO dojo_findingvulnerabilityreference (finding_id, vulnerability_id, "order", created) SELECT s.finding_id, e.id, s.ord - 1, now() @@ -98,9 +137,10 @@ def _stamp_missing_types(cursor, logger): def run_backfill(*, logger=None, window_size=DEFAULT_WINDOW_SIZE): """ - Run the full idempotent backfill. PostgreSQL only. Returns a dict of per-step row counts. + Run the full idempotent, convergent backfill. PostgreSQL only. Returns per-step row counts. - Callers (migration 0286 / migrate_vulnerability_ids) are responsible for the vendor guard. + The reference build is convergent (per-window DELETE + re-insert in one transaction each), so it + is safe to re-run as a deploy-time catch-up after a pre-run even when ids changed in between. """ logger = logger or logging.getLogger(__name__) counts = {} @@ -115,17 +155,26 @@ def run_backfill(*, logger=None, window_size=DEFAULT_WINDOW_SIZE): counts["entities_from_cve"] = cursor.rowcount counts["types_stamped"] = _stamp_missing_types(cursor, logger) - logger.info("Backfill step 3/3: references (windowed by finding_id, window=%s)", window_size) - cursor.execute("SELECT MIN(finding_id), MAX(finding_id) FROM dojo_vulnerability_id") + logger.info("Backfill step 3/3: references (convergent, windowed by finding_id, window=%s)", window_size) + cursor.execute(_SELECT_REFERENCE_WINDOW_BOUNDS) lo, hi_max = cursor.fetchone() - references = 0 + deleted = 0 + inserted = 0 if lo is not None: while lo <= hi_max: hi = lo + window_size - cursor.execute(_INSERT_REFERENCES_WINDOW, {"lo": lo, "hi": hi}) - references += cursor.rowcount + # One transaction per window: the DELETE + INSERT commit together, so a crash leaves + # the window either fully rebuilt or untouched (resumable), and the DELETE's row lock + # serializes the rebuild against a concurrent live delete-then-insert on the same + # finding. atomic=False on the migration keeps each window a separate commit. + with transaction.atomic(): + cursor.execute(_DELETE_REFERENCES_WINDOW, {"lo": lo, "hi": hi}) + deleted += cursor.rowcount + cursor.execute(_INSERT_REFERENCES_WINDOW, {"lo": lo, "hi": hi}) + inserted += cursor.rowcount lo = hi - counts["references_from_legacy"] = references + counts["references_deleted"] = deleted + counts["references_from_legacy"] = inserted logger.info("Backfill complete: %s", counts) return counts diff --git a/dojo/vulnerability/manager.py b/dojo/vulnerability/manager.py index e8bb628d17e..28828b94ae2 100644 --- a/dojo/vulnerability/manager.py +++ b/dojo/vulnerability/manager.py @@ -2,8 +2,8 @@ Write seam for vulnerability ids: the ``Vulnerability`` entity + ``FindingVulnerabilityReference`` rows are the single source of truth. Every writer persists them through this one module. -The legacy ``Vulnerability_Id`` store is no longer written (entity-only cutover); the legacy table -still exists but is frozen until it is dropped in a follow-up migration. +The legacy ``Vulnerability_Id`` store is gone: this entity-only cutover stops writing it and drops +its table in the same change (migration 0288, after the 0287 backfill). CWE buffering (store_cwes/pending_cwes/reconcile_cwes) is deliberately NOT owned here — it keeps riding the importer flush boundary exactly as before. """ diff --git a/dojo/vulnerability/queries.py b/dojo/vulnerability/queries.py index 2c36902324e..23f46b70041 100644 --- a/dojo/vulnerability/queries.py +++ b/dojo/vulnerability/queries.py @@ -18,7 +18,8 @@ def get_auth_filter(key): # --------------------------------------------------------------------------- # Read helpers over the entity/reference store — the single source of truth. # (The legacy Vulnerability_Id read path + V3_FEATURE_VULNERABILITY_IDS flag were -# removed in the entity-only cutover; the legacy table is frozen until dropped.) +# removed in the entity-only cutover; the legacy table is dropped by migration 0288 +# in the same change, after the 0287 backfill.) # --------------------------------------------------------------------------- From d59681b87ed5ced17a164b71d161444ab69a3275 Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:14:18 -0600 Subject: [PATCH 17/18] docs(vuln-id): remove stale dual-write/flag comments (entity-only) The compat fields and reconcile/copy paths still described the removed read-flag + dual-write architecture. Reword to the entity-only reality: VulnerabilityIdsField reads the entity references; copy()/reimport read through the entity helper and write through the entity seam; the unique_finding_order comment says "write path" not "dual-write". Co-Authored-By: Claude Opus 4.8 (1M context) --- dojo/finding/api/serializer.py | 10 +++++----- dojo/finding/models.py | 7 +++---- dojo/importers/default_reimporter.py | 9 ++++----- dojo/vulnerability/models.py | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/dojo/finding/api/serializer.py b/dojo/finding/api/serializer.py index 5d4b19b33ab..ea4b8d0aee3 100644 --- a/dojo/finding/api/serializer.py +++ b/dojo/finding/api/serializer.py @@ -309,10 +309,10 @@ class VulnerabilityIdsField(serializers.Field): """ Wire-frozen v2 vulnerability_ids field. - Reads ``[{"vulnerability_id": str}]`` from the flag-appropriate store (legacy rows or the - entity references) — byte-identical either way because writes are dual. Accepts the same - object list (tolerating bare strings) on write and hands the parsed strings to create/update - under ``parsed_vulnerability_ids``, which funnel to save_vulnerability_ids (unchanged path). + Reads ``[{"vulnerability_id": str}]`` from the finding's entity references (ordered, + primary/cve first). Accepts the same object list (tolerating bare strings) on write and hands + the parsed strings to create/update under ``parsed_vulnerability_ids``, which funnel to + save_vulnerability_ids (unchanged path). """ def __init__(self, **kwargs): @@ -325,7 +325,7 @@ def get_attribute(self, instance): return instance def to_representation(self, finding): - from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- flag seam + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- avoid import cycle return [{"vulnerability_id": value} for value in finding_vulnerability_id_strings(finding)] def to_internal_value(self, data): diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 6469118d879..cab4be3bb21 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -705,10 +705,9 @@ def copy(self, test=None): # Assign any tags copy.tags.set(old_tags) # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util). - # Route vulnerability ids through the dual-write seam so the copy gets legacy rows AND - # entity references (copy.cve was already carried over by copy_model_util). Read the SOURCE - # ids through the flag seam too (not the legacy relation directly), so copy() keeps working - # after the legacy Vulnerability_Id store is retired. + # Write the copy's ids through the entity write seam (copy.cve was already carried over by + # copy_model_util), and read the SOURCE ids through the entity read helper (not a legacy + # relation). from dojo.vulnerability.manager import persist_for_finding # noqa: PLC0415 -- avoid import cycle from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index cffd8eb8e10..26f27fcd5d5 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -988,10 +988,9 @@ def reconcile_vulnerability_ids( # while vulnerability_ids do not, and vice versa). self.reconcile_cwes(finding) - # Read the existing ids through the flag seam (entity references when the flag is on, legacy - # rows when off) instead of the legacy relation directly, so reconcile keeps working after - # the legacy Vulnerability_Id store is retired. The prefetch is seam-matched upstream (the - # reimport finding query uses vulnerability_id_prefetch()), so this stays a no-query read. + # Read the existing ids through the entity read helper (not a legacy relation). The prefetch + # is matched upstream (the reimport finding query uses vulnerability_id_prefetch()), so this + # stays a no-query read. from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- avoid import cycle existing_vuln_ids = set(finding_vulnerability_id_strings(finding)) @@ -1005,7 +1004,7 @@ def reconcile_vulnerability_ids( ) return finding - # Accumulate delete + insert for batch flush (legacy rows + entity refs, dual-write). + # Accumulate delete + insert for batch flush (entity references). self.vulnerability_id_manager.record_reconcile(finding, vulnerability_ids_to_process) if vulnerability_ids_to_process: finding.cve = vulnerability_ids_to_process[0] diff --git a/dojo/vulnerability/models.py b/dojo/vulnerability/models.py index aa00f45ae8f..4399ac51f92 100644 --- a/dojo/vulnerability/models.py +++ b/dojo/vulnerability/models.py @@ -77,7 +77,7 @@ class Meta: models.UniqueConstraint(fields=["finding", "vulnerability"], name="unique_finding_vulnerability"), # Structural guarantee of a single primary id per finding: no two references may share # an order (in particular, never two order-0 rows). Compatible with the delete-then- - # insert dual-write, which assigns 0..n-1 in one transaction (see manager.py). + # insert write path, which assigns 0..n-1 in one transaction (see manager.py). models.UniqueConstraint(fields=["finding", "order"], name="unique_finding_order"), ] indexes = [models.Index(fields=["finding", "order"]), models.Index(fields=["vulnerability"])] From e334025d45881d33055eeeb3ac1521c226926fdb Mon Sep 17 00:00:00 2001 From: Cody Maffucci <46459665+Maffooch@users.noreply.github.com> Date: Fri, 24 Jul 2026 10:44:06 -0600 Subject: [PATCH 18/18] feat(vuln-id): retain legacy Vulnerability_Id model+table for a transition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defer the destructive drop. Instead of the entity-only cutover dropping dojo_vulnerability_id in the same release, keep the legacy Vulnerability_Id model and table for a transition period and remove them together in a future release, once the entity store has proven itself in production. - Restore the Vulnerability_Id model (unused: no code path reads or writes it; entity references are the source of truth) + its dojo.models re-export. - Delete migration 0288 (no drop, no SeparateDatabaseAndState state/DB split, which complicated other tooling). The model stays in Django state, matching code — makemigrations --check + manage.py check are clean. - The physical DROP TABLE moves to a future release; guard it there with the invariant (abort if legacy has rows while the reference table is empty). Validated: fresh migrate from zero through 0287 keeps both table sets; merge, search, and vulnerability-id suites green. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../0288_delete_vulnerability_id.py | 47 ------------------- dojo/finding/models.py | 42 +++++++++++++++-- dojo/models.py | 1 + dojo/vulnerability/manager.py | 5 +- dojo/vulnerability/queries.py | 5 +- 5 files changed, 46 insertions(+), 54 deletions(-) delete mode 100644 dojo/db_migrations/0288_delete_vulnerability_id.py diff --git a/dojo/db_migrations/0288_delete_vulnerability_id.py b/dojo/db_migrations/0288_delete_vulnerability_id.py deleted file mode 100644 index 5145c462c98..00000000000 --- a/dojo/db_migrations/0288_delete_vulnerability_id.py +++ /dev/null @@ -1,47 +0,0 @@ -# Entity-only cutover: drop the legacy Vulnerability_Id store. Vulnerability ids now live entirely in -# the Vulnerability entity + FindingVulnerabilityReference through-table (created in 0286, backfilled -# from dojo_vulnerability_id in 0287 — which runs BEFORE this drop). ⚠️ DESTRUCTIVE: drops -# dojo_vulnerability_id and its data. -# -# Safety invariant (the useful residue of the removed verify_vulnerability_ids command): refuse to -# drop the legacy table if it still holds rows the backfill never transferred. This is the last -# moment such a check can act — once the table is gone, the source of truth is gone with it. It -# catches the one catastrophic ordering mistake (0288 reached with 0287 skipped/failed, e.g. on a -# non-PostgreSQL backend where the raw-SQL backfill never ran) and turns silent data loss into a -# loud, actionable migration failure. - -from django.db import connection, migrations - -_ABORT_MESSAGE = ( - "Refusing to drop dojo_vulnerability_id: it still has rows but " - "dojo_findingvulnerabilityreference is empty, so the 0287 backfill did not run (or did not " - "complete). Dropping now would permanently lose vulnerability-id data. Run " - "`manage.py migrate_vulnerability_ids` (PostgreSQL only) to backfill, then re-run migrate." -) - - -def assert_backfill_ran(apps, schema_editor): - """Abort the drop if the legacy table has data but the reference table is empty.""" - with connection.cursor() as cursor: - cursor.execute("SELECT EXISTS (SELECT 1 FROM dojo_vulnerability_id)") - legacy_has_rows = cursor.fetchone()[0] - cursor.execute("SELECT EXISTS (SELECT 1 FROM dojo_findingvulnerabilityreference)") - references_exist = cursor.fetchone()[0] - if legacy_has_rows and not references_exist: - raise RuntimeError(_ABORT_MESSAGE) - - -class Migration(migrations.Migration): - - dependencies = [ - ("dojo", "0287_backfill_vulnerability_id_entities"), - ] - - operations = [ - # Guard runs before the drop; on an atomic migration a raised RuntimeError rolls the whole - # migration back, so the DeleteModel never executes when the invariant is violated. - migrations.RunPython(assert_backfill_ran, migrations.RunPython.noop), - migrations.DeleteModel( - name="Vulnerability_Id", - ), - ] diff --git a/dojo/finding/models.py b/dojo/finding/models.py index cab4be3bb21..87a2448bee0 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -25,6 +25,7 @@ from dojo.base_models.base import BaseModel from dojo.finding.cwe import cwe_label, finding_cwe_labels +from dojo.finding.vulnerability_id import resolve_vulnerability_id_type # get_current_date/tomorrow/copy_model_util are defined early in dojo.models, before the # re-export that loads this module — so this resolves despite the partial circular load, and @@ -1438,9 +1439,44 @@ def set_hash_code(self, dedupe_option): deduplicationLogger.debug("Hash_code computed for finding: %s: %s", finding_id, self.hash_code) -# The legacy Vulnerability_Id model was removed in the entity-only cutover; vulnerability ids now -# live in the Vulnerability entity + FindingVulnerabilityReference through-table (dojo/vulnerability/). -# The dojo_vulnerability_id table is dropped by migration 0287. +# Legacy vulnerability-id store. As of the entity-only cutover, vulnerability ids live in the +# Vulnerability entity + FindingVulnerabilityReference through-table (dojo/vulnerability/), and this +# model is NO LONGER READ OR WRITTEN by any code path. It (and its dojo_vulnerability_id table) are +# intentionally RETAINED for a transition period as a frozen pre-cutover snapshot, and removed +# together in a future release. Do not add new usages. +class Vulnerability_Id(models.Model): + finding = models.ForeignKey("dojo.Finding", editable=False, on_delete=models.CASCADE) + vulnerability_id = models.TextField(max_length=50, blank=False, null=False) + # Autodetected from the id prefix (CVE, GHSA, ...); NULL when there is no non-numeric + # prefix. Denormalized/indexed so type-scoped queries (e.g. GROUP BY type) stay cheap. + vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, editable=False, db_index=True) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=["finding", "vulnerability_id"], name="unique_finding_vulnerability_id"), + ] + indexes = [ + # Leading on vulnerability_id (the unique constraint's index leads on finding), for the + # vulnerability-id Explorer's GROUP BY vulnerability_id / lookups by exact id. + models.Index(fields=["vulnerability_id"], name="dojo_vuln_id_lookup_idx"), + # Global search (pro/search/): weighted tsvector FTS + trigram fuzzy match. + GinIndex( + SearchVector("vulnerability_id", weight="A", config="english"), + name="dojo_vulnerability_id_fts_gin", + ), + GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vuln_id_trgm"), + ] + + def __str__(self): + return self.vulnerability_id + + def save(self, *args, **kwargs): + # bulk_create paths set the type at construction; this covers save()/get_or_create. + self.vulnerability_id_type = resolve_vulnerability_id_type(self.vulnerability_id) + super().save(*args, **kwargs) + + def get_absolute_url(self): + return reverse("view_finding", args=[str(self.finding.id)]) class Finding_CWE(models.Model): diff --git a/dojo/models.py b/dojo/models.py index 42a001d78dd..0a8f72b21d9 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -405,6 +405,7 @@ class Meta: Finding_CWE, # noqa: F401 -- re-export Finding_Group, # noqa: F401 -- re-export Finding_Template, + Vulnerability_Id, # noqa: F401 -- re-export (legacy; retained for a transition period, see model) ) from dojo.vulnerability.models import ( # noqa: E402 -- re-export; FKs reference dojo.Finding / dojo.Vulnerability by string FindingVulnerabilityReference, # noqa: F401 -- re-export diff --git a/dojo/vulnerability/manager.py b/dojo/vulnerability/manager.py index 28828b94ae2..12ae5e1450b 100644 --- a/dojo/vulnerability/manager.py +++ b/dojo/vulnerability/manager.py @@ -2,8 +2,9 @@ Write seam for vulnerability ids: the ``Vulnerability`` entity + ``FindingVulnerabilityReference`` rows are the single source of truth. Every writer persists them through this one module. -The legacy ``Vulnerability_Id`` store is gone: this entity-only cutover stops writing it and drops -its table in the same change (migration 0288, after the 0287 backfill). +The legacy ``Vulnerability_Id`` store is no longer written or read after this entity-only cutover. +Its model and ``dojo_vulnerability_id`` table are retained (unused) for a transition period as a +frozen pre-cutover snapshot, and removed together in a future release. CWE buffering (store_cwes/pending_cwes/reconcile_cwes) is deliberately NOT owned here — it keeps riding the importer flush boundary exactly as before. """ diff --git a/dojo/vulnerability/queries.py b/dojo/vulnerability/queries.py index 23f46b70041..780c1739d51 100644 --- a/dojo/vulnerability/queries.py +++ b/dojo/vulnerability/queries.py @@ -18,8 +18,9 @@ def get_auth_filter(key): # --------------------------------------------------------------------------- # Read helpers over the entity/reference store — the single source of truth. # (The legacy Vulnerability_Id read path + V3_FEATURE_VULNERABILITY_IDS flag were -# removed in the entity-only cutover; the legacy table is dropped by migration 0288 -# in the same change, after the 0287 backfill.) +# removed in the entity-only cutover; the legacy Vulnerability_Id model + table are +# retained unused for a transition period after the 0287 backfill and removed +# together in a future release.) # ---------------------------------------------------------------------------