feat(search): add DD_WATSON_SEARCH_ENABLED toggle to gate watson indexing (#15236)

Master switch (default True, so OSS behaviour is unchanged). When False: watson
models are not registered (no post-save index writes), the watson app +
AsyncSearchContextMiddleware are stripped from settings, /simple_search returns
410, and installwatson is skipped (both the complete_initialization command and
entrypoint-first-boot.sh).

Lets deployments that have replaced watson (e.g. DefectDojo Pro's native Postgres
search) stop paying the indexing cost and reclaim watson_searchentry.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: blakeaowens

docker/entrypoint-first-boot.sh

@@ -27,8 +27,13 @@ EOD
     python3 manage.py loaddata "${i%.*}"
   done
 
-  echo "Installing watson search index"
-  python3 manage.py installwatson
+  watson_enabled=$(echo "${DD_WATSON_SEARCH_ENABLED:-True}" | tr '[:upper:]' '[:lower:]')
+  if [ "$watson_enabled" != "false" ] && [ "$watson_enabled" != "0" ] && [ "$watson_enabled" != "off" ] && [ "$watson_enabled" != "no" ]; then
+    echo "Installing watson search index"
+    python3 manage.py installwatson
+  else
+    echo "Skipping watson search index (DD_WATSON_SEARCH_ENABLED=${DD_WATSON_SEARCH_ENABLED})"
+  fi
 
   # surveys fixture needs to be modified as it contains an instance dependant polymorphic content id
   echo "Migration of textquestions for surveys"

dojo/apps.py

@@ -1,6 +1,7 @@
 import logging
 
 from django.apps import AppConfig
+from django.conf import settings
 from django.core.checks import register as register_check
 from django.db import models
 from watson import search as watson
@@ -21,54 +22,11 @@ def ready(self):
         # commented out ^ as it prints in manage.py dumpdata, docker logs and many other places
         # logger doesn't work yet at this stage
 
-        # Watson doesn't have a way to let it index extra fields, so we have to explicitly list all the fields
-        # to make it easier, we get the charfields/textfields from the model and then add our extra fields.
-        # charfields/textfields are the fields that watson indexes by default (but we have to repeat here if we add extra fields)
-        # and watson likes to have tuples instead of lists
-
-        watson.register(self.get_model("Product"), fields=get_model_fields_with_extra(self.get_model("Product"), ("id", "prod_type__name")), store=("prod_type__name", ))
-
-        watson.register(self.get_model("Test"), fields=get_model_fields_with_extra(self.get_model("Test"), ("id", "engagement__product__name")), store=("engagement__product__name", ))  # test_type__name?
-
-        watson.register(self.get_model("Finding"), fields=get_model_fields_with_extra(self.get_model("Finding"), ("id", "url", "unique_id_from_tool", "test__engagement__product__name", "jira_issue__jira_key")),
-                        store=("status", "jira_issue__jira_key", "test__engagement__product__name", "severity", "severity_display", "latest_note"))
-
-        # some thoughts on Finding fields that are not indexed yet:
-        # CWE can't be indexed as it is an integer
-
-        # would endpoints be good to index? or would it clutter search results?
-        # endpoints = models.ManyToManyField(Endpoint, blank=True)
-        # endpoint_status = models.ManyToManyField(Endpoint_Status, blank=True, related_name='finding_endpoint_status')
-
-        # index test name/title?
-        # test = models.ForeignKey(Test, editable=False, on_delete=models.CASCADE)
-
-        # index reporter name?
-        # reporter = models.ForeignKey(User, editable=False, default=1, related_name='reporter', on_delete=models.CASCADE)
-        # index notes?
-        # notes = models.ManyToManyField(Notes, blank=True, editable=False)
-
-        # index found_by?
-        # found_by = models.ManyToManyField(Test_Type, editable=False)
-
-        # exclude these to avoid cluttering?
-        # sast_source_object = models.CharField(null=True, blank=True, max_length=500, help_text="Source object (variable, function...) of the attack vector")
-        # sast_sink_object = models.CharField(null=True, blank=True, max_length=500, help_text="Sink object (variable, function...) of the attack vector")
-        # sast_source_line = models.IntegerField(null=True, blank=True,
-        #                            verbose_name="Line number",
-        #                            help_text="Source line number of the attack vector")
-        # sast_source_file_path = models.CharField(null=True, blank=True, max_length=4000, help_text="Source filepath of the attack vector")
-
-        watson.register(self.get_model("Finding_Template"))
-        # TODO: Delete this after the move to Locations
-        watson.register(self.get_model("Endpoint"), store=("product__name", ))  # add product name also?
-        watson.register(self.get_model("Location"))
-        watson.register(self.get_model("Engagement"), fields=get_model_fields_with_extra(self.get_model("Engagement"), ("id", "product__name")), store=("product__name", ))
-        watson.register(self.get_model("App_Analysis"))
-        watson.register(self.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", ))
-
-        # YourModel = self.get_model("YourModel")
-        # watson.register(YourModel)
+        # Registering watson attaches post-save/pre-delete search indexers to 9
+        # models (Finding is on the import hot path); skip it entirely when watson
+        # is disabled so imports don't pay the write-amplification cost.
+        if settings.WATSON_SEARCH_ENABLED:
+            register_watson_models(self)
 
         register_check(check_configuration_deduplication, "dojo")
 
@@ -110,6 +68,57 @@ def ready(self):
         install_intermediate_flush_hook()
 
 
+def register_watson_models(app_config):
+    # Watson doesn't have a way to let it index extra fields, so we have to explicitly list all the fields
+    # to make it easier, we get the charfields/textfields from the model and then add our extra fields.
+    # charfields/textfields are the fields that watson indexes by default (but we have to repeat here if we add extra fields)
+    # and watson likes to have tuples instead of lists
+
+    watson.register(app_config.get_model("Product"), fields=get_model_fields_with_extra(app_config.get_model("Product"), ("id", "prod_type__name")), store=("prod_type__name", ))
+
+    watson.register(app_config.get_model("Test"), fields=get_model_fields_with_extra(app_config.get_model("Test"), ("id", "engagement__product__name")), store=("engagement__product__name", ))  # test_type__name?
+
+    watson.register(app_config.get_model("Finding"), fields=get_model_fields_with_extra(app_config.get_model("Finding"), ("id", "url", "unique_id_from_tool", "test__engagement__product__name", "jira_issue__jira_key")),
+                    store=("status", "jira_issue__jira_key", "test__engagement__product__name", "severity", "severity_display", "latest_note"))
+
+    # some thoughts on Finding fields that are not indexed yet:
+    # CWE can't be indexed as it is an integer
+
+    # would endpoints be good to index? or would it clutter search results?
+    # endpoints = models.ManyToManyField(Endpoint, blank=True)
+    # endpoint_status = models.ManyToManyField(Endpoint_Status, blank=True, related_name='finding_endpoint_status')
+
+    # index test name/title?
+    # test = models.ForeignKey(Test, editable=False, on_delete=models.CASCADE)
+
+    # index reporter name?
+    # reporter = models.ForeignKey(User, editable=False, default=1, related_name='reporter', on_delete=models.CASCADE)
+    # index notes?
+    # notes = models.ManyToManyField(Notes, blank=True, editable=False)
+
+    # index found_by?
+    # found_by = models.ManyToManyField(Test_Type, editable=False)
+
+    # exclude these to avoid cluttering?
+    # sast_source_object = models.CharField(null=True, blank=True, max_length=500, help_text="Source object (variable, function...) of the attack vector")
+    # sast_sink_object = models.CharField(null=True, blank=True, max_length=500, help_text="Sink object (variable, function...) of the attack vector")
+    # sast_source_line = models.IntegerField(null=True, blank=True,
+    #                            verbose_name="Line number",
+    #                            help_text="Source line number of the attack vector")
+    # sast_source_file_path = models.CharField(null=True, blank=True, max_length=4000, help_text="Source filepath of the attack vector")
+
+    watson.register(app_config.get_model("Finding_Template"))
+    # TODO: Delete this after the move to Locations
+    watson.register(app_config.get_model("Endpoint"), store=("product__name", ))  # add product name also?
+    watson.register(app_config.get_model("Location"))
+    watson.register(app_config.get_model("Engagement"), fields=get_model_fields_with_extra(app_config.get_model("Engagement"), ("id", "product__name")), store=("product__name", ))
+    watson.register(app_config.get_model("App_Analysis"))
+    watson.register(app_config.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", ))
+
+    # YourModel = app_config.get_model("YourModel")
+    # watson.register(YourModel)
+
+
 def get_model_fields_with_extra(model, extra_fields=()):
     return get_model_fields(get_model_default_fields(model), extra_fields)
 

dojo/management/commands/complete_initialization.py

@@ -138,7 +138,8 @@ def first_boot_setup(self) -> None:
         self.load_initial_fixtures()
         self.persist_jira_webhook_secret()
         self.load_extra_fixtures()
-        self.install_watson()
+        if settings.WATSON_SEARCH_ENABLED:
+            self.install_watson()
         call_command("migrate_textquestions")
 
     def create_admin_user(self) -> None:

dojo/search/views.py

@@ -5,6 +5,7 @@
 
 from django.conf import settings
 from django.db.models import Q
+from django.http import HttpResponseGone
 from django.shortcuts import render
 from django.utils.translation import gettext as _
 from watson import search as watson
@@ -67,6 +68,12 @@ def simple_search(request):
     operators: {'tags': ['anchore'], 'vulnerability_id': ['CVE-2020-1234']}
     keywords:  ['jquery']
     """
+    if not settings.WATSON_SEARCH_ENABLED:
+        # Legacy watson-backed search is retired: return 410 rather than query
+        # models that are no longer registered with watson. Pro overrides this
+        # route with a redirect to its native search UI.
+        return HttpResponseGone("Legacy search has been disabled.")
+
     tests = None
     findings = None
     finding_templates = None

dojo/settings/settings.dist.py

@@ -125,6 +125,13 @@
     DD_TAG_BULK_ADD_BATCH_SIZE=(int, 1000),
     # Tagulous slug truncate unique setting. Set to -1 to use tagulous internal default (5)
     DD_TAGULOUS_SLUG_TRUNCATE_UNIQUE=(int, -1),
+    # Master switch for django-watson. When True (default) the post-save/pre-delete
+    # search indexers are registered on 9 models (write-amplification on every save,
+    # Finding imports especially) and the legacy /simple_search page (watson's only
+    # reader) is served. When False, nothing is registered, /simple_search returns
+    # 410 Gone, and installwatson is skipped. Pro ships this False: its native
+    # Postgres search (pro/search/) fully replaces watson.
+    DD_WATSON_SEARCH_ENABLED=(bool, True),
     # Batch size for async watson search-index update tasks. Also doubles as
     # the per-request intermediate-flush threshold: once the in-memory watson
     # context reaches this many pending objects mid-request,
@@ -848,6 +855,15 @@ def generate_url(scheme, double_slashes, user, password, host, port, path, param
     ]
     MIDDLEWARE += WHITE_NOISE
 
+# django-watson master switch (see DD_WATSON_SEARCH_ENABLED above). Pro ships this
+# off because its native Postgres search replaces watson; disabling strips the app
+# and its request-scoped indexing middleware so no post-save index writes happen
+# and installwatson is never needed.
+WATSON_SEARCH_ENABLED = env("DD_WATSON_SEARCH_ENABLED")
+if not WATSON_SEARCH_ENABLED:
+    INSTALLED_APPS = tuple(app for app in INSTALLED_APPS if app != "watson")
+    MIDDLEWARE = [m for m in MIDDLEWARE if m != "dojo.middleware.AsyncSearchContextMiddleware"]
+
 EMAIL_CONFIG = env.email_url(
     "DD_EMAIL_URL", default="smtp://user@:password@localhost:25")
 

unittests/test_watson_search_disabled.py

@@ -0,0 +1,44 @@
+from django.contrib.auth import get_user_model
+from django.test import override_settings
+from django.urls import reverse
+
+from dojo.models import System_Settings
+from unittests.dojo_test_case import DojoTestCase
+
+User = get_user_model()
+
+
+# Regression: watson_searchentry grew to 44 GB while nothing in the Pro UI reads it.
+# The legacy /simple_search page is watson's only reader, so when watson is disabled
+# (DD_WATSON_SEARCH_ENABLED=False) the view must stop serving instead of calling
+# watson.filter/search on models that were never registered.
+#
+# V3_FEATURE_LOCATIONS is pinned off so this matches OSS CI defaults (the deprecated
+# Endpoint model raises under V3, which is unrelated to what we assert here).
+@override_settings(SECURE_SSL_REDIRECT=False, V3_FEATURE_LOCATIONS=False)
+class TestSimpleSearchWatsonToggle(DojoTestCase):
+
+    def setUp(self):
+        System_Settings.objects.get_or_create(id=1)
+        super().setUp()
+        self.user = User.objects.create_superuser("watson-toggle-admin", "wt@example.com", "password")
+
+    def _get_simple_search(self):
+        self.client.force_login(self.user)
+        return self.client.get(reverse("simple_search"), {"query": "test"})
+
+    @override_settings(WATSON_SEARCH_ENABLED=False)
+    def test_simple_search_gone_when_watson_disabled(self):
+        response = self._get_simple_search()
+        self.assertEqual(
+            response.status_code, 410,
+            msg=f"expected 410 Gone when WATSON_SEARCH_ENABLED=False, got {response.status_code}",
+        )
+
+    @override_settings(WATSON_SEARCH_ENABLED=True)
+    def test_simple_search_available_when_watson_enabled(self):
+        response = self._get_simple_search()
+        self.assertEqual(
+            response.status_code, 200,
+            msg=f"expected 200 when WATSON_SEARCH_ENABLED=True, got {response.status_code}",
+        )