feat: [five-c] Create rules-based engine #409 (#552)

Signed-off-by: tdruez <tdruez@aboutcode.org>

Author: tdruez

dejacode/settings.py

@@ -475,7 +475,9 @@ def gettext_noop(s):
 
 # Cron jobs (scheduler)
 daily_at_3am = "0 3 * * *"
+hourly = "0 * * * *"
 DEJACODE_VULNERABILITIES_CRON = env.str("DEJACODE_VULNERABILITIES_CRON", default=daily_at_3am)
+DEJACODE_POLICY_RULES_CRON = env.str("DEJACODE_POLICY_RULES_CRON", default=hourly)
 
 
 def enable_rq_eager_mode():

dje/admin.py

@@ -96,6 +96,7 @@
 from dje.views import manage_tab_permissions_view
 from dje.views import object_compare_view
 from dje.views import object_copy_view
+from policy.rules import RULE_REGISTRY
 
 EXTERNAL_SOURCE_LOOKUP = "external_references__external_source_id"
 
@@ -1046,12 +1047,26 @@ def render(self, name, value, attrs=None, renderer=None):
 
 
 class DataspaceConfigurationForm(forms.ModelForm):
-    """
-    Configure Dataspace settings.
+    """Configure Dataspace integration settings, with sensitive values hidden in the UI."""
 
-    This form includes fields for various API keys, with sensitive values
-    hidden in the UI using the HiddenValueWidget.
-    """
+    class Meta:
+        model = DataspaceConfiguration
+        fields = [
+            "homepage_layout",
+            "scancodeio_url",
+            "scancodeio_api_key",
+            "vulnerablecode_url",
+            "vulnerablecode_api_key",
+            "vulnerabilities_risk_threshold",
+            "purldb_url",
+            "purldb_api_key",
+            "forgejo_token",
+            "github_token",
+            "gitlab_token",
+            "jira_user",
+            "jira_token",
+            "sourcehut_token",
+        ]
 
     hidden_value_fields = [
         "scancodeio_api_key",
@@ -1078,6 +1093,73 @@ def clean(self):
                 del self.cleaned_data[field_name]
 
 
+class PolicyRulesConfigurationForm(forms.ModelForm):
+    """Form for configuring policy rule overrides stored in policy_rules_config."""
+
+    class Meta:
+        model = DataspaceConfiguration
+        fields = []
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.add_policy_rule_config_fields()
+
+    def add_policy_rule_config_fields(self):
+        """Inject per-rule form fields with initial values from the saved policy_rules_config."""
+        config = getattr(self.instance, "policy_rules_config", {}) or {}
+        for rule_type, handler in RULE_REGISTRY.items():
+            rule_config = config.get(rule_type, {})
+            self.fields[f"rule_{rule_type}_enabled"] = forms.BooleanField(
+                label="Enable this rule",
+                required=False,
+                initial=rule_config.get("is_active", False),
+            )
+            self.fields[f"rule_{rule_type}_threshold"] = forms.IntegerField(
+                label="Threshold",
+                required=False,
+                min_value=0,
+                initial=rule_config.get("threshold"),
+                widget=forms.NumberInput(
+                    attrs={"placeholder": f"Default: {handler.default_threshold}"}
+                ),
+                help_text="Minimum violations to trigger the rule. Leave blank to use the default.",
+            )
+            for param_name, param_desc in handler.parameters_schema.items():
+                self.fields[f"rule_{rule_type}_param_{param_name}"] = forms.FloatField(
+                    label=param_name.replace("_", " ").title(),
+                    required=False,
+                    initial=(rule_config.get("parameters") or {}).get(param_name),
+                    help_text=param_desc,
+                )
+
+    def build_policy_rules_config(self):
+        """Serialize the per-rule form fields back into the policy_rules_config dict."""
+        policy_rules_config = {}
+        for rule_type, handler in RULE_REGISTRY.items():
+            rule_config = {}
+            if self.cleaned_data.get(f"rule_{rule_type}_enabled"):
+                rule_config["is_active"] = True
+            threshold = self.cleaned_data.get(f"rule_{rule_type}_threshold")
+            if threshold is not None:
+                rule_config["threshold"] = threshold
+            parameters = {}
+            for param_name in handler.parameters_schema:
+                param_value = self.cleaned_data.get(f"rule_{rule_type}_param_{param_name}")
+                if param_value is not None:
+                    parameters[param_name] = param_value
+            if parameters:
+                rule_config["parameters"] = parameters
+            if rule_config:
+                policy_rules_config[rule_type] = rule_config
+        return policy_rules_config
+
+    def save(self, commit=True):
+        self.instance.policy_rules_config = self.build_policy_rules_config()
+        if commit:
+            self.instance.save(update_fields=["policy_rules_config"])
+        return self.instance
+
+
 class DataspaceConfigurationInline(DataspacedFKMixin, admin.StackedInline):
     model = DataspaceConfiguration
     form = DataspaceConfigurationForm
@@ -1142,12 +1224,13 @@ class DataspaceConfigurationInline(DataspacedFKMixin, admin.StackedInline):
     ]
     # Do not include the Dataspace related FKs on addition as the Dataspace does not exist yet
     fieldsets = [("", {"fields": ("homepage_layout",)})] + add_fieldsets
+    inline_classes = ("grp-collapse grp-open",)
     can_delete = False
 
     def get_fieldsets(self, request, obj=None):
         if not obj:
             return self.add_fieldsets
-        return super().get_fieldsets(request, obj)
+        return [("", {"fields": ("homepage_layout",)})] + self.add_fieldsets
 
     def get_readonly_fields(self, request, obj=None):
         """Only a user from the current Dataspace can edit Dataspace related FKs."""
@@ -1160,6 +1243,40 @@ def get_readonly_fields(self, request, obj=None):
         return readonly_fields
 
 
+class PolicyRulesConfigurationInline(DataspacedFKMixin, admin.StackedInline):
+    model = DataspaceConfiguration
+    form = PolicyRulesConfigurationForm
+    verbose_name_plural = _("Policy Rules Configuration")
+    verbose_name = _("Policy Rules Configuration")
+    classes = ("grp-collapse grp-open",)
+    inline_classes = ("grp-collapse grp-open",)
+    can_delete = False
+
+    def get_fieldsets(self, request, obj=None):
+        if not obj:
+            return []
+        rule_fieldsets = []
+        for rule_type, handler in RULE_REGISTRY.items():
+            fields = [f"rule_{rule_type}_enabled", f"rule_{rule_type}_threshold"]
+            for param_name in handler.parameters_schema:
+                fields.append(f"rule_{rule_type}_param_{param_name}")
+            rule_fieldsets.append(
+                (
+                    handler.label,
+                    {
+                        "fields": fields,
+                        "description": handler.description,
+                        "classes": ("grp-collapse grp-open",),
+                    },
+                )
+            )
+        return rule_fieldsets
+
+    def get_formset(self, request, obj=None, **kwargs):
+        kwargs["fields"] = []
+        return super().get_formset(request, obj, **kwargs)
+
+
 @admin.register(Dataspace, site=dejacode_site)
 class DataspaceAdmin(
     ReferenceOnlyPermissions,
@@ -1239,7 +1356,7 @@ class DataspaceAdmin(
         ),
     )
     search_fields = ("name",)
-    inlines = [DataspaceConfigurationInline]
+    inlines = [DataspaceConfigurationInline, PolicyRulesConfigurationInline]
     form = DataspaceAdminForm
     change_form_template = "admin/dje/dataspace/change_form.html"
     change_list_template = "admin/change_list_extended.html"

dje/cron_jobs.py

@@ -11,6 +11,7 @@
 from rq import cron
 
 from dje.tasks import update_vulnerabilities
+from policy.tasks import evaluate_all_products_rules_task
 
 two_hour = 7200
 
@@ -20,3 +21,10 @@
     cron=settings.DEJACODE_VULNERABILITIES_CRON,  # Daily at 3am by default
     job_timeout=two_hour,
 )
+
+cron.register(
+    func=evaluate_all_products_rules_task,
+    queue_name="default",
+    cron=settings.DEJACODE_POLICY_RULES_CRON,  # Hourly by default
+    job_timeout=two_hour,
+)

dje/migrations/0016_dataspaceconfiguration_policy_rules_config.py

@@ -0,0 +1,18 @@
+# Generated by Django 6.0.6 on 2026-07-15 08:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dje', '0015_alter_dataspaceconfiguration_purldb_api_key_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='dataspaceconfiguration',
+            name='policy_rules_config',
+            field=models.JSONField(blank=True, default=dict, help_text='Override default policy rule settings for this dataspace.'),
+        ),
+    ]

dje/models.py

@@ -614,6 +614,12 @@ class DataspaceConfiguration(DataspaceForeignKeyValidationMixin, models.Model):
         ),
     )
 
+    policy_rules_config = models.JSONField(
+        blank=True,
+        default=dict,
+        help_text=_("Override default policy rule settings for this dataspace."),
+    )
+
     def __str__(self):
         return f"{self.dataspace}"
 

dje/tests/test_admin.py

@@ -13,10 +13,12 @@
 from django.test import override_settings
 from django.urls import reverse
 
+from dje.admin import PolicyRulesConfigurationForm
 from dje.copier import copy_object
 from dje.filters import DataspaceFilter
 from dje.filters import MissingInFilter
 from dje.models import Dataspace
+from dje.models import DataspaceConfiguration
 from dje.models import History
 from dje.search import advanced_search
 from dje.tests import add_perm
@@ -25,6 +27,7 @@
 from dje.tests import create_superuser
 from dje.tests import create_user
 from organization.models import Owner
+from policy.rules import RULE_REGISTRY
 
 
 class DataspacedModelAdminTestCase(TestCase):
@@ -178,6 +181,8 @@ def test_dataspace_admin_changeform_update_packages_from_scan_field_validation(s
             "update_packages_from_scan": True,
             "configuration-TOTAL_FORMS": 0,
             "configuration-INITIAL_FORMS": 0,
+            "configuration-2-TOTAL_FORMS": 0,
+            "configuration-2-INITIAL_FORMS": 0,
         }
 
         response = self.client.post(url, data)
@@ -611,3 +616,78 @@ def test_admin_group_permission_export_csv(self):
             'attachment; filename="dejacode_group_permission.csv"', response["Content-Disposition"]
         )
         self.assertEqual(b",change_license\r\nchange license,X\r\n", response.content)
+
+
+class PolicyRulesConfigurationFormTestCase(TestCase):
+    def setUp(self):
+        self.dataspace = Dataspace.objects.create(name="nexB")
+        self.config = DataspaceConfiguration.objects.create(dataspace=self.dataspace)
+        self.Form = PolicyRulesConfigurationForm
+
+    def _bound_form(self, extra_data=None):
+        data = {}
+        for rule_type in RULE_REGISTRY:
+            data[f"rule_{rule_type}_enabled"] = False
+            data[f"rule_{rule_type}_threshold"] = ""
+        if extra_data:
+            data.update(extra_data)
+        return self.Form(data=data, instance=self.config)
+
+    def test_enabled_rule_included_in_config(self):
+        form = self._bound_form({"rule_usage_policy_error_enabled": True})
+        self.assertTrue(form.is_valid(), form.errors)
+        result = form.build_policy_rules_config()
+        self.assertIn("usage_policy_error", result)
+        self.assertTrue(result["usage_policy_error"]["is_active"])
+
+    def test_disabled_rule_not_in_config(self):
+        form = self._bound_form()
+        self.assertTrue(form.is_valid(), form.errors)
+        result = form.build_policy_rules_config()
+        self.assertEqual({}, result)
+
+    def test_threshold_saved_when_set(self):
+        form = self._bound_form(
+            {
+                "rule_usage_policy_error_enabled": True,
+                "rule_usage_policy_error_threshold": "3",
+            }
+        )
+        self.assertTrue(form.is_valid(), form.errors)
+        result = form.build_policy_rules_config()
+        self.assertEqual(3, result["usage_policy_error"]["threshold"])
+
+    def test_initial_values_loaded_from_existing_config(self):
+        self.config.policy_rules_config = {
+            "usage_policy_error": {"is_active": True, "threshold": 7}
+        }
+        self.config.save()
+        form = self.Form(instance=self.config)
+        self.assertTrue(form.fields["rule_usage_policy_error_enabled"].initial)
+        self.assertEqual(7, form.fields["rule_usage_policy_error_threshold"].initial)
+        self.assertFalse(form.fields["rule_license_coverage_gap_enabled"].initial)
+
+    def test_save_persists_policy_rules_config_to_db(self):
+        form = self._bound_form({"rule_usage_policy_error_enabled": True})
+        self.assertTrue(form.is_valid(), form.errors)
+        form.save()
+        self.config.refresh_from_db()
+        self.assertTrue(self.config.policy_rules_config["usage_policy_error"]["is_active"])
+
+    def test_inline_shows_rule_fieldsets_on_existing_dataspace(self):
+        self.super_user = create_superuser("super_user", self.dataspace)
+        self.client.login(username="super_user", password="secret")
+        url = reverse("admin:dje_dataspace_change", args=[self.dataspace.pk])
+        response = self.client.get(url)
+        self.assertEqual(200, response.status_code)
+        self.assertContains(response, "Policy Rules Configuration")
+        for rule_type in RULE_REGISTRY:
+            self.assertContains(response, f"rule_{rule_type}_enabled")
+
+    def test_inline_not_shown_on_dataspace_add(self):
+        self.super_user = create_superuser("super_user", self.dataspace)
+        self.client.login(username="super_user", password="secret")
+        url = reverse("admin:dje_dataspace_add")
+        response = self.client.get(url)
+        self.assertEqual(200, response.status_code)
+        self.assertNotContains(response, "rule_usage_policy_error_enabled")

dje/tests/test_history.py

@@ -58,6 +58,8 @@ def test_history_on_admin_dataspace_add(self):
             "name": "new_dataspace",
             "configuration-TOTAL_FORMS": 0,
             "configuration-INITIAL_FORMS": 0,
+            "configuration-2-TOTAL_FORMS": 0,
+            "configuration-2-INITIAL_FORMS": 0,
         }
 
         self.client.post(url, params)

notification/admin.py

@@ -64,3 +64,9 @@ class WebhookSubscriptionAdmin(ProhibitDataspaceLookupMixin, DataspacedAdmin):
     actions_to_remove = ["copy_to", "compare_with"]
     email_notification_on = ()
     inlines = [WebhookDeliveryInline]
+
+    def get_inlines(self, request, obj=None):
+        """Exclude delivery history on the add form as no deliveries exist yet."""
+        if obj is None:
+            return []
+        return super().get_inlines(request, obj)

notification/models.py

@@ -28,6 +28,8 @@
     "user.added_or_updated",
     "user.locked_out",
     "vulnerability.data_update",
+    "policy.violation_detected",
+    "policy.violation_resolved",
 ]
 
 

notification/tasks.py

@@ -12,6 +12,7 @@
 
 from django_rq import job
 
+from dje.models import get_unsecured_manager
 from notification.models import WebhookSubscription
 
 logger = logging.getLogger("dje")
@@ -36,7 +37,7 @@ def deliver_webhook_task(
     if instance_app_label and instance_model_name and instance_pk:
         try:
             model_class = apps.get_model(instance_app_label, instance_model_name)
-            instance = model_class.objects.get(pk=instance_pk)
+            instance = get_unsecured_manager(model_class).get(pk=instance_pk)
         except Exception:
             logger.error(
                 f"Instance {instance_app_label}.{instance_model_name} pk={instance_pk} not found."