perf(tags): drop inherited_tags TagField, use _inherited_tag_names JSONField (Phase B Stage 3)

dojo/api_v2/serializers.py

@@ -1110,7 +1110,7 @@ class EngagementSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Engagement
-        exclude = ("inherited_tags",)
+        exclude = ("_inherited_tag_names",)
 
     def validate(self, data):
         if self.context["request"].method == "POST":
@@ -1282,7 +1282,7 @@ class EndpointSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Endpoint
-        exclude = ("inherited_tags",)
+        exclude = ("_inherited_tag_names",)
 
     def validate(self, data):
 
@@ -1418,7 +1418,7 @@ class TestSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Test
-        exclude = ("inherited_tags",)
+        exclude = ("_inherited_tag_names",)
 
     def build_relational_field(self, field_name, relation_info):
         if field_name == "notes":
@@ -1442,7 +1442,7 @@ class TestCreateSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = Test
-        exclude = ("inherited_tags",)
+        exclude = ("_inherited_tag_names",)
 
 
 class TestTypeCreateSerializer(serializers.ModelSerializer):
@@ -1746,7 +1746,7 @@ class Meta:
         model = Finding
         exclude = (
             "cve",
-            "inherited_tags",
+            "_inherited_tag_names",
         )
 
     # TODO: Delete this after the move to Locations
@@ -1967,7 +1967,7 @@ class Meta:
         model = Finding
         exclude = (
             "cve",
-            "inherited_tags",
+            "_inherited_tag_names",
         )
         extra_kwargs = {
             "active": {"required": True},

dojo/auditlog/backfill.py

@@ -57,30 +57,18 @@ def get_table_names(model_name):
     elif model_name == "FindingTags":
         table_name = "dojo_finding_tags"
         event_table_name = "dojo_finding_tagsevent"
-    elif model_name == "FindingInheritedTags":
-        table_name = "dojo_finding_inherited_tags"
-        event_table_name = "dojo_finding_inherited_tagsevent"
     elif model_name == "ProductTags":
         table_name = "dojo_product_tags"
         event_table_name = "dojo_product_tagsevent"
     elif model_name == "EngagementTags":
         table_name = "dojo_engagement_tags"
         event_table_name = "dojo_engagement_tagsevent"
-    elif model_name == "EngagementInheritedTags":
-        table_name = "dojo_engagement_inherited_tags"
-        event_table_name = "dojo_engagement_inherited_tagsevent"
     elif model_name == "TestTags":
         table_name = "dojo_test_tags"
         event_table_name = "dojo_test_tagsevent"
-    elif model_name == "TestInheritedTags":
-        table_name = "dojo_test_inherited_tags"
-        event_table_name = "dojo_test_inherited_tagsevent"
     elif model_name == "EndpointTags":
         table_name = "dojo_endpoint_tags"
         event_table_name = "dojo_endpoint_tagsevent"
-    elif model_name == "EndpointInheritedTags":
-        table_name = "dojo_endpoint_inherited_tags"
-        event_table_name = "dojo_endpoint_inherited_tagsevent"
     elif model_name == "FindingTemplateTags":
         table_name = "dojo_finding_template_tags"
         event_table_name = "dojo_finding_template_tagsevent"

dojo/auditlog/services.py

@@ -398,15 +398,15 @@ class Meta:
     # the through models, which can't be imported at module level.
     tag_through_models = [
         # (Parent model, field name, proxy class name)
+        # `inherited_tags` TagField was removed in the tag-inheritance
+        # redesign; the inherited-name list now lives in a
+        # `_inherited_tag_names` JSONField whose changes are captured by the
+        # parent table's pghistory event automatically.
         (Finding, "tags", "FindingTags"),
-        (Finding, "inherited_tags", "FindingInheritedTags"),
         (Product, "tags", "ProductTags"),
         (Engagement, "tags", "EngagementTags"),
-        (Engagement, "inherited_tags", "EngagementInheritedTags"),
         (Test, "tags", "TestTags"),
-        (Test, "inherited_tags", "TestInheritedTags"),
         (Endpoint, "tags", "EndpointTags"),
-        (Endpoint, "inherited_tags", "EndpointInheritedTags"),
         (Finding_Template, "tags", "FindingTemplateTags"),
         (App_Analysis, "tags", "AppAnalysisTags"),
         (Objects_Product, "tags", "ObjectsProductTags"),

dojo/db_migrations/0265_inherited_tag_names_jsonfield.py

@@ -0,0 +1,124 @@
+"""
+Replace the duplicate `inherited_tags` TagField on Engagement / Test /
+Finding / Endpoint / Location with a `_inherited_tag_names` JSONField.
+
+Phase B Stage 3 of the tag inheritance redesign. Copies existing M2M data
+into the JSON column, then drops the M2M field (which also drops the
+auto-generated through tables and the Tagulous tag tables for the
+inherited_tags side).
+"""
+from django.db import migrations, models
+
+
+def copy_inherited_tags_to_json(apps, schema_editor):
+    """Copy each child's inherited_tags M2M values into _inherited_tag_names JSON."""
+    for app_label, model_name in [
+        ("dojo", "Engagement"),
+        ("dojo", "Test"),
+        ("dojo", "Finding"),
+        ("dojo", "Endpoint"),
+        ("dojo", "Location"),
+    ]:
+        try:
+            Model = apps.get_model(app_label, model_name)
+        except LookupError:
+            continue
+        for obj in Model.objects.iterator(chunk_size=1000):
+            try:
+                names = sorted(obj.inherited_tags.values_list("name", flat=True))
+            except Exception:
+                names = []
+            if names:
+                Model.objects.filter(pk=obj.pk).update(_inherited_tag_names=names)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("dojo", "0264_alter_url_identity_hash_alter_urlevent_identity_hash"),
+    ]
+
+    operations = [
+        # 1. Add the JSON column to each child model.
+        migrations.AddField(
+            model_name="engagement",
+            name="_inherited_tag_names",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Internal: tag names inherited from the product, used to identify which entries in `tags` came from inheritance vs user input.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="endpoint",
+            name="_inherited_tag_names",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Internal: tag names inherited from the product, used to identify which entries in `tags` came from inheritance vs user input.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="test",
+            name="_inherited_tag_names",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Internal: tag names inherited from the product, used to identify which entries in `tags` came from inheritance vs user input.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="finding",
+            name="_inherited_tag_names",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Internal: tag names inherited from the product, used to identify which entries in `tags` came from inheritance vs user input.",
+            ),
+        ),
+        migrations.AddField(
+            model_name="location",
+            name="_inherited_tag_names",
+            field=models.JSONField(
+                blank=True,
+                default=list,
+                help_text="Internal: tag names inherited from the product, used to identify which entries in `tags` came from inheritance vs user input.",
+            ),
+        ),
+        # 2. Copy existing M2M data into the JSON column.
+        migrations.RunPython(copy_inherited_tags_to_json, migrations.RunPython.noop),
+        # 3. Drop pghistory proxies and event-tracking tables for the
+        #    inherited_tags through tables (created by migration 0256).
+        #    Must precede the RemoveField below: Django's state-rendering
+        #    fails to resolve the proxy bases once their through table
+        #    target is gone.
+        migrations.DeleteModel(name="EngagementInheritedTagsEvent"),
+        migrations.DeleteModel(name="EndpointInheritedTagsEvent"),
+        migrations.DeleteModel(name="TestInheritedTagsEvent"),
+        migrations.DeleteModel(name="FindingInheritedTagsEvent"),
+        migrations.DeleteModel(name="EngagementInheritedTags"),
+        migrations.DeleteModel(name="EndpointInheritedTags"),
+        migrations.DeleteModel(name="TestInheritedTags"),
+        migrations.DeleteModel(name="FindingInheritedTags"),
+        # 4. Drop the duplicate inherited_tags TagField on each child. Django
+        #    will also drop the auto-generated `dojo_<model>_inherited_tags`
+        #    through tables.
+        migrations.RemoveField(model_name="engagement", name="inherited_tags"),
+        migrations.RemoveField(model_name="endpoint", name="inherited_tags"),
+        migrations.RemoveField(model_name="test", name="inherited_tags"),
+        migrations.RemoveField(model_name="finding", name="inherited_tags"),
+        migrations.RemoveField(model_name="location", name="inherited_tags"),
+        # 5. Drop the now-orphaned Tagulous tag models that backed the
+        #    `inherited_tags` TagFields. These were created in migration
+        #    0188 (and 0259 for Location).
+        migrations.DeleteModel(name="Tagulous_Engagement_inherited_tags"),
+        migrations.DeleteModel(name="Tagulous_Endpoint_inherited_tags"),
+        migrations.DeleteModel(name="Tagulous_Test_inherited_tags"),
+        migrations.DeleteModel(name="Tagulous_Finding_inherited_tags"),
+        migrations.DeleteModel(name="Tagulous_Location_inherited_tags"),
+        # No GIN index added: the current code reads the JSON column per
+        # row via `_sync_inheritance_for_qs` (Python-side diff) rather than
+        # filtering with `_inherited_tag_names__contains`. Add a GIN index
+        # in a follow-up if production query patterns shift toward SQL-side
+        # containment lookups.
+    ]