feat(vuln-id): flag-gated entity read seam, wire-compatible

Reads switch to the VulnerabilityId entity/reference store when
V3_FEATURE_VULNERABILITY_IDS is on (default True); off keeps today's legacy-table
reads. The flag is consulted ONLY in dojo/vulnerability_id/queries.py and the two
Finding methods. Writes stay unconditionally dual (WP3), so both stores hold
identical data and the flip is reversible with no drift.

queries.py seam: use_entity_reads, finding_ids_with_vulnerability_ids (in/exact),
vulnerability_id_prefetch (with nested prefix), first_vulnerability_id_subquery
(order==0), finding_vulnerability_id_strings.

Re-pointed (identical output flag-off; entity read flag-on):
- Finding.vulnerability_ids property + get_vulnerability_ids() (cve prepend /
  sorted-join kept byte-for-byte).
- dojo/filters.py both filter fns; risk_acceptance exact-match mixin.
- The 7 prefetch sites (deduplication, commands/dedupe, finding/ui + test/ui
  views, finding/queries x2) -> vulnerability_id_prefetch().
- v2 serializer: VulnerabilityIdsField reads [{"vulnerability_id": s}] from the
  flag-appropriate store (schema pinned via extend_schema_field); create/update
  pop `parsed_vulnerability_ids` and still funnel writes to save_vulnerability_ids.
- Finding.copy() now dual-writes copied ids through persist_for_finding (was a
  legacy-only .objects.create — would have been invisible flag-on).

Deliberately NOT re-pointed this PR (kept on the dual-written legacy table, which
stays correct in both flag states; entity migration deferred to the legacy-drop
phase): watson registration + the classic OSS search view. Rationale: watson is
disabled in the Pro runtime (untestable locally), the classic search view/template
are coupled to watson indexing Vulnerability_Id, and the classic UI is being
retired. This also removes the buildwatson-on-flip requirement. The Pro Vue global
search IS re-pointed to the entity in the Pro PR.

Tests (fixture-free, run locally + CI): test_vulnerability_id.TestReadSeamFlagParity
asserts the v2 wire shape, hash input (get_vulnerability_ids), the property, and the
filter seam are byte-identical flag-off vs flag-on; test_importers_importer reconcile
seeds via the dual-write seam so entity reads resolve. Verified green in BOTH flag
states locally; the fixture-based suites (rest_framework schema, import_reimport,
deduplication_logic, finding_helper API, watson trio) run in OSS CI.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Maffooch

dojo/apps.py

@@ -113,6 +113,10 @@ def register_watson_models(app_config):
     watson.register(app_config.get_model("Location"))
     watson.register(app_config.get_model("Engagement"), fields=get_model_fields_with_extra(app_config.get_model("Engagement"), ("id", "product__name")), store=("product__name", ))
     watson.register(app_config.get_model("App_Analysis"))
+    # Watson stays on the legacy Vulnerability_Id table (unconditionally dual-written), so global
+    # search keeps working identically in both flag states with no buildwatson reindex. Moving the
+    # watson index (and the classic search view) to FindingVulnerabilityReference is a legacy-drop
+    # phase concern, not this reads-only flag flip.
     watson.register(app_config.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", ))
 
     # YourModel = app_config.get_model("YourModel")

dojo/filters.py

@@ -52,10 +52,10 @@
     Product,
     Product_Type,
     Test,
-    Vulnerability_Id,
 )
 from dojo.product_type.queries import get_authorized_product_types
 from dojo.utils import get_system_setting, is_finding_groups_enabled, truncate_timezone_aware
+from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids
 
 logger = logging.getLogger(__name__)
 
@@ -73,17 +73,11 @@ def custom_filter(queryset, name, value):
 
 def custom_vulnerability_id_filter(queryset, name, value):
     values = value.split(",")
-    ids = Vulnerability_Id.objects \
-        .filter(vulnerability_id__in=values) \
-        .values_list("finding_id", flat=True)
-    return queryset.filter(id__in=ids)
+    return queryset.filter(id__in=finding_ids_with_vulnerability_ids(values, lookup="in"))
 
 
 def vulnerability_id_filter(queryset, name, value):
-    ids = Vulnerability_Id.objects \
-        .filter(vulnerability_id=value) \
-        .values_list("finding_id", flat=True)
-    return queryset.filter(id__in=ids)
+    return queryset.filter(id__in=finding_ids_with_vulnerability_ids(value, lookup="exact"))
 
 
 class NumberInFilter(filters.BaseInFilter, filters.NumberFilter):

dojo/finding/api/serializer.py

@@ -301,6 +301,51 @@ class Meta:
         fields = ["vulnerability_id"]
 
 
+@extend_schema_field(VulnerabilityIdSerializer(many=True))
+class VulnerabilityIdsField(serializers.Field):
+
+    """
+    Wire-frozen v2 vulnerability_ids field.
+
+    Reads ``[{"vulnerability_id": str}]`` from the flag-appropriate store (legacy rows or the
+    entity references) — byte-identical either way because writes are dual. Accepts the same
+    object list (tolerating bare strings) on write and hands the parsed strings to create/update
+    under ``parsed_vulnerability_ids``, which funnel to save_vulnerability_ids (unchanged path).
+    """
+
+    def __init__(self, **kwargs):
+        kwargs["source"] = "*"
+        kwargs.setdefault("required", False)
+        super().__init__(**kwargs)
+
+    def get_attribute(self, instance):
+        # source="*" -> to_representation receives the whole Finding.
+        return instance
+
+    def to_representation(self, finding):
+        from dojo.vulnerability_id.queries import finding_vulnerability_id_strings  # noqa: PLC0415 -- flag seam
+        return [{"vulnerability_id": value} for value in finding_vulnerability_id_strings(finding)]
+
+    def to_internal_value(self, data):
+        if not isinstance(data, list):
+            msg = "Expected a list of vulnerability ids."
+            raise serializers.ValidationError(msg)
+        parsed = []
+        for item in data:
+            if isinstance(item, dict):
+                if "vulnerability_id" not in item:
+                    msg = 'Each vulnerability id object requires a "vulnerability_id" key.'
+                    raise serializers.ValidationError(msg)
+                parsed.append(item["vulnerability_id"])
+            elif isinstance(item, str):
+                parsed.append(item)
+            else:
+                msg = 'Each vulnerability id must be a string or {"vulnerability_id": "..."}.'
+                raise serializers.ValidationError(msg)
+        # source="*" merges this dict into validated_data; create/update pop the key.
+        return {"parsed_vulnerability_ids": parsed}
+
+
 @extend_schema_field(serializers.CharField())
 class CweField(serializers.Field):
 
@@ -345,9 +390,7 @@ class FindingSerializer(serializers.ModelSerializer):
     finding_groups = FindingGroupSerializer(
         source="finding_group_set", many=True, read_only=True,
     )
-    vulnerability_ids = VulnerabilityIdSerializer(
-        source="vulnerability_id_set", many=True, required=False,
-    )
+    vulnerability_ids = VulnerabilityIdsField(required=False)
     cwes = FindingCweSerializer(
         source="finding_cwe_set", many=True, required=False,
     )
@@ -439,12 +482,10 @@ def update(self, instance, validated_data):
         # push_all_issues already checked in api views.py
         push_to_jira = validated_data.pop("push_to_jira")
 
-        # Save vulnerability ids and pop them
-        parsed_vulnerability_ids = []
-        if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)):
-            logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids)
-            parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids)
-            logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0])
+        # Save vulnerability ids and pop them (VulnerabilityIdsField parsed them to strings)
+        parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or []
+        if parsed_vulnerability_ids:
+            logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0])
             validated_data["cve"] = parsed_vulnerability_ids[0]
 
         # CWEs (mirror vulnerability_ids): the first entry is the primary Finding.cwe; the rest
@@ -603,9 +644,7 @@ class FindingCreateSerializer(serializers.ModelSerializer):
     )
     url = serializers.CharField(allow_null=True, default=None)
     push_to_jira = serializers.BooleanField(default=False)
-    vulnerability_ids = VulnerabilityIdSerializer(
-        source="vulnerability_id_set", many=True, required=False,
-    )
+    vulnerability_ids = VulnerabilityIdsField(required=False)
     cwes = FindingCweSerializer(
         source="finding_cwe_set", many=True, required=False,
     )
@@ -639,15 +678,12 @@ def create(self, validated_data):
         notes = validated_data.pop("notes", None)
         found_by = validated_data.pop("found_by", None)
         reviewers = validated_data.pop("reviewers", None)
-        # Process the vulnerability IDs specially
-        parsed_vulnerability_ids = []
-        if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)):
-            logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids)
-            parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids)
-            logger.debug("PARSED_VULNERABILITY_IDST: %s", parsed_vulnerability_ids)
-            logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0])
+        # Process the vulnerability IDs specially (VulnerabilityIdsField parsed them to strings)
+        parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or []
+        if parsed_vulnerability_ids:
+            logger.debug("PARSED_VULNERABILITY_IDS: %s", parsed_vulnerability_ids)
+            logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0])
             validated_data["cve"] = parsed_vulnerability_ids[0]
-            # validated_data["unsaved_vulnerability_ids"] = parsed_vulnerability_ids
 
         # CWEs (mirror vulnerability_ids): first entry is the primary cwe, the rest are extras.
         parsed_cwes = None

dojo/finding/deduplication.py

@@ -9,6 +9,7 @@
 
 from dojo.celery import app
 from dojo.models import Endpoint_Status, Finding, System_Settings
+from dojo.vulnerability_id.queries import vulnerability_id_prefetch
 
 logger = logging.getLogger(__name__)
 deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication")
@@ -340,11 +341,11 @@ def build_candidate_scope_queryset(test, mode="deduplication", service=None):
         queryset = Finding.objects.filter(scope_q)
 
     if settings.V3_FEATURE_LOCATIONS:
-        prefetch_list = ["locations__location__url", "vulnerability_id_set", "finding_cwe_set", "found_by"]
+        prefetch_list = ["locations__location__url", vulnerability_id_prefetch(), "finding_cwe_set", "found_by"]
     else:
         # TODO: Delete this after the move to Locations
         # Base prefetches for both modes
-        prefetch_list = ["endpoints", "vulnerability_id_set", "finding_cwe_set", "found_by"]
+        prefetch_list = ["endpoints", vulnerability_id_prefetch(), "finding_cwe_set", "found_by"]
 
         # Prefetch all endpoint statuses with their endpoint for reimport mode.
         # The non-special filtering (excluding false_positive, out_of_scope, risk_accepted)

dojo/finding/models.py

@@ -705,9 +705,13 @@ def copy(self, test=None):
         copy.found_by.set(old_found_by)
         # Assign any tags
         copy.tags.set(old_tags)
-        # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util)
-        for vulnerability_id in self.vulnerability_id_set.all():
-            Vulnerability_Id.objects.create(finding=copy, vulnerability_id=vulnerability_id.vulnerability_id)
+        # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util).
+        # Route vulnerability ids through the dual-write seam so the copy gets legacy rows AND
+        # entity references (copy.cve was already carried over by copy_model_util).
+        from dojo.vulnerability_id.manager import persist_for_finding  # noqa: PLC0415 -- avoid import cycle
+        vulnerability_id_strings = [row.vulnerability_id for row in self.vulnerability_id_set.all()]
+        if vulnerability_id_strings:
+            persist_for_finding(copy, vulnerability_id_strings, delete_existing=False)
         for finding_cwe in self.finding_cwe_set.all():
             Finding_CWE.objects.create(finding=copy, cwe=finding_cwe.cwe)
 
@@ -838,12 +842,18 @@ def _get_unsaved_vulnerability_ids(finding) -> str:
 
         def _get_saved_vulnerability_ids(finding) -> str:
             if finding.id is not None:
-                # Use the reverse relation (vulnerability_id_set) rather than a fresh
-                # Vulnerability_Id.objects.filter(...) so prefetch_related("vulnerability_id_set")
-                # is honored — avoids an N+1 (COUNT + SELECT per finding) during dedupe/hashcode.
-                vulnerability_id_str_list = [str(vulnerability_id) for vulnerability_id in finding.vulnerability_id_set.all()]
+                from dojo.vulnerability_id.queries import use_entity_reads  # noqa: PLC0415
+                if use_entity_reads():
+                    # Entity store: the prefetch-honoring reverse relation (vulnerability_references
+                    # + select_related("vulnerability")) so Prefetch is honored — no N+1 in dedupe.
+                    vulnerability_id_str_list = [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()]
+                else:
+                    # Use the reverse relation (vulnerability_id_set) rather than a fresh
+                    # Vulnerability_Id.objects.filter(...) so prefetch_related("vulnerability_id_set")
+                    # is honored — avoids an N+1 (COUNT + SELECT per finding) during dedupe/hashcode.
+                    vulnerability_id_str_list = [str(vulnerability_id) for vulnerability_id in finding.vulnerability_id_set.all()]
                 deduplicationLogger.debug("get_vulnerability_ids after the finding was saved. Vulnerability references count: " + str(len(vulnerability_id_str_list)))
-                # sort vulnerability_ids strings
+                # sort vulnerability_ids strings (no dedupe — byte parity with the legacy store)
                 return "".join(sorted(vulnerability_id_str_list))
             return ""
 
@@ -1383,9 +1393,14 @@ def get_references_with_links(self):
 
     @cached_property
     def vulnerability_ids(self):
+        from dojo.vulnerability_id.queries import use_entity_reads  # noqa: PLC0415
         # Get vulnerability ids from database and convert to list of strings
-        vulnerability_ids_model = self.vulnerability_id_set.all()
-        vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model]
+        if use_entity_reads():
+            # Entity store: reverse relation is ordered by FindingVulnerabilityReference.Meta.ordering.
+            vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in self.vulnerability_references.all()]
+        else:
+            vulnerability_ids_model = self.vulnerability_id_set.all()
+            vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model]
 
         # Synchronize the cve field with the unsaved_vulnerability_ids
         # We do this to be as flexible as possible to handle the fields until

dojo/finding/queries.py

@@ -20,6 +20,7 @@ def get_auth_filter(key): return None
     Vulnerability_Id,
 )
 from dojo.request_cache import cache_for_request_or_task
+from dojo.vulnerability_id.queries import vulnerability_id_prefetch
 
 logger = logging.getLogger(__name__)
 
@@ -111,7 +112,7 @@ def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=Tr
             "status_finding",
             "finding_group_set",
             "finding_group_set__jira_issue",  # Include both variants
-            "vulnerability_id_set",
+            vulnerability_id_prefetch(),
         )
         base_status = LocationFindingReference.objects.prefetch_related("location__url").all()
         prefetched_findings = prefetched_findings.annotate(
@@ -147,7 +148,7 @@ def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=Tr
             "status_finding",
             "finding_group_set",
             "finding_group_set__jira_issue",  # Include both variants
-            "vulnerability_id_set",
+            vulnerability_id_prefetch(),
         )
         base_status = Endpoint_Status.objects.prefetch_related("endpoint")
         status = Case(

dojo/finding/ui/views.py

@@ -121,6 +121,7 @@
     reopen_external_issue,
     update_external_issue,
 )
+from dojo.vulnerability_id.queries import vulnerability_id_prefetch
 
 JFORM_PUSH_TO_JIRA_MESSAGE = "jform.push_to_jira: %s"
 
@@ -168,7 +169,7 @@ def prefetch_for_similar_findings(findings):
         prefetched_findings = prefetched_findings.prefetch_related("notes")
         prefetched_findings = prefetched_findings.prefetch_related("tags")
         prefetched_findings = prefetched_findings.prefetch_related(
-            "vulnerability_id_set",
+            vulnerability_id_prefetch(),
         )
     else:
         logger.debug("unable to prefetch because query was already executed")

dojo/management/commands/dedupe.py

@@ -20,6 +20,7 @@
     get_system_setting,
     mass_model_updater,
 )
+from dojo.vulnerability_id.queries import vulnerability_id_prefetch
 
 logger = logging.getLogger(__name__)
 deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication")
@@ -97,10 +98,10 @@ def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedup
                 "test", "test__engagement", "test__engagement__product", "test__test_type",
             ).prefetch_related(
                 "locations",
-                # vulnerability_id_set feeds hash_code computation for parsers whose
+                # vulnerability id store feeds hash_code computation for parsers whose
                 # HASHCODE_FIELDS_PER_SCANNER includes vulnerability_ids; prefetch to avoid
                 # a per-finding query in get_vulnerability_ids().
-                "vulnerability_id_set",
+                vulnerability_id_prefetch(),
                 Prefetch(
                     "original_finding",
                     queryset=Finding.objects.only("id", "duplicate_finding_id").order_by("-id"),
@@ -113,10 +114,10 @@ def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedup
                 "test", "test__engagement", "test__engagement__product", "test__test_type",
             ).prefetch_related(
                 "endpoints",
-                # vulnerability_id_set feeds hash_code computation for parsers whose
+                # vulnerability id store feeds hash_code computation for parsers whose
                 # HASHCODE_FIELDS_PER_SCANNER includes vulnerability_ids; prefetch to avoid
                 # a per-finding query in get_vulnerability_ids().
-                "vulnerability_id_set",
+                vulnerability_id_prefetch(),
                 Prefetch(
                     "original_finding",
                     queryset=Finding.objects.only("id", "duplicate_finding_id").order_by("-id"),

dojo/risk_acceptance/api/mixins.py

@@ -12,8 +12,9 @@
 
 from dojo.authorization.api_permissions import UserHasRiskAcceptanceRelatedObjectPermission
 from dojo.engagement.queries import get_authorized_engagements
-from dojo.models import Engagement, Risk_Acceptance, User, Vulnerability_Id
+from dojo.models import Engagement, Risk_Acceptance, User
 from dojo.risk_acceptance.api.serializer import RiskAcceptanceSerializer
+from dojo.vulnerability_id.queries import finding_ids_with_vulnerability_ids
 
 AcceptedRisk = NamedTuple("AcceptedRisk", (("vulnerability_id", str), ("justification", str), ("accepted_by", str)))
 
@@ -90,10 +91,8 @@ def accept_risks(self, request):
 def _accept_risks(accepted_risks: list[AcceptedRisk], base_findings: QuerySet, owner: User):
     accepted = []
     for risk in accepted_risks:
-        vulnerability_ids = Vulnerability_Id.objects \
-            .filter(vulnerability_id=risk.vulnerability_id) \
-            .values("finding")
-        findings = base_findings.filter(id__in=vulnerability_ids)
+        finding_ids = finding_ids_with_vulnerability_ids(risk.vulnerability_id, lookup="exact")
+        findings = base_findings.filter(id__in=finding_ids)
         if findings.exists():
             # TODO: we could use risk.vulnerability_id to name the risk_acceptance, but would need to check for existing risk_acceptances in that case
             # so for now we add some timestamp based suffix

dojo/test/ui/views.py

@@ -82,6 +82,7 @@
     process_tag_notifications,
     redirect_to_return_url_or_else,
 )
+from dojo.vulnerability_id.queries import vulnerability_id_prefetch
 
 logger = logging.getLogger(__name__)
 parse_logger = logging.getLogger("dojo")
@@ -172,7 +173,7 @@ def get_initial_context(self, request: HttpRequest, test: Test):
             "jira_project": jira_services.get_project(test),
             "bulk_edit_form": FindingBulkUpdateForm(request.GET),
             "enable_table_filtering": get_system_setting("enable_ui_table_based_searching"),
-            "finding_groups": test.finding_group_set.all().prefetch_related("findings", "jira_issue", "creator", "findings__vulnerability_id_set"),
+            "finding_groups": test.finding_group_set.all().prefetch_related("findings", "jira_issue", "creator", vulnerability_id_prefetch(prefix="findings__")),
             "finding_group_by_options": Finding_Group.GROUP_BY_OPTIONS,
         }
         # Set the form using the context, and then update the context