auditlog: switch to pghistory

Author: valentijnscholten

.github/workflows/integration-tests.yml

@@ -2,18 +2,12 @@ name: Integration tests
 
 on:
   workflow_call:
-    inputs:
-        auditlog_type:
-          type: string
-          default: "django-auditlog"
 
 jobs:
   integration_tests:
     # run tests with docker compose
     name: User Interface Tests
     runs-on: ubuntu-latest
-    env:
-      AUDITLOG_TYPE: ${{ inputs.auditlog_type }}
     strategy:
       matrix:
         test-case: [

.github/workflows/rest-framework-tests.yml

@@ -6,16 +6,11 @@ on:
         platform:
           type: string
           default: "linux/amd64"
-        auditlog_type:
-          type: string
-          default: "django-auditlog"
 
 jobs:
   unit_tests:
     name: Rest Framework Unit Tests
     runs-on: ${{ inputs.platform ==  'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
-    env:
-      AUDITLOG_TYPE: ${{ inputs.auditlog_type }}
 
     strategy:
       matrix:

.github/workflows/unit-tests.yml

@@ -25,26 +25,18 @@ jobs:
     strategy:
         matrix:
             platform: ['linux/amd64', 'linux/arm64']
-            auditlog_type: ['django-auditlog', 'django-pghistory']
         fail-fast: false
     needs: build-docker-containers
     uses: ./.github/workflows/rest-framework-tests.yml
     secrets: inherit
     with:
       platform: ${{ matrix.platform}}
-      auditlog_type: ${{ matrix.auditlog_type }}
 
   # only run integration tests for linux/amd64 (default)
   test-user-interface:
     needs: build-docker-containers
     uses: ./.github/workflows/integration-tests.yml
     secrets: inherit
-    strategy:
-      matrix:
-        auditlog_type: ['django-auditlog', 'django-pghistory']
-      fail-fast: false
-    with:
-      auditlog_type: ${{ matrix.auditlog_type }}
 
   # only run k8s tests for linux/amd64 (default)
   test-k8s:

docs/content/en/open_source/upgrading/2.53.md

@@ -0,0 +1,38 @@
+---
+title: "Upgrading to DefectDojo Version 2.53.x"
+toc_hide: true
+weight: -20251101
+description: Removal of django-auditlog and exclusive use of django-pghistory for audit logging.
+---
+
+## Breaking Change: Removal of django-auditlog
+
+Starting with DefectDojo 2.53, `django-auditlog` support has been removed in favour of `django-pghistory`.
+This is designed to be a backwards compatible change, unless:
+- You're querying the database directly for auditlog events, or,
+- You've set the `DD_AUDITLOG_TYPE` environment variable (or `AUDITLOG_TYPE` settings field)
+
+### Required Actions
+
+If you're using `DD_AUDITLOG_TYPE`, remove it from your configuration/environment.
+
+### Existing Records Preserved
+
+Historical audit log entries stored in the `auditlog_logentry` table will continue to be displayed in the action history view for backward compatibility. No data migration is required.
+
+### Benefits of django-pghistory
+
+The switch to `django-pghistory` provides several advantages:
+
+- **Better performance**: Database-level triggers reduce overhead compared to Django signal-based auditing
+- **More features**: Enhanced context tracking and better support for complex queries
+- **Better data integrity**: PostgreSQL-native implementation ensures consistency
+
+### Migration Notes
+
+- A one-time data migration will take place to populate the `django-pghistory` tables with the initial snapshot of the tracked models.
+
+---
+
+Check the [Release Notes](https://github.com/DefectDojo/django-DefectDojo/releases/tag/2.53.0) for the complete contents of this release.
+

dojo/apps.py

@@ -91,7 +91,8 @@ def ready(self):
         # Configure audit system after all models are loaded
         # This must be done in ready() to avoid "Models aren't loaded yet" errors
         # Note: pghistory models are registered here (no database access), but trigger
-        # enabling is handled via management command to avoid database access warnings
+        # enabling is handled in the entrpoint script to avoid database access warnings
+        # during startup
         register_django_pghistory_models()
         configure_audit_system()
 

dojo/auditlog.py

@@ -1,11 +1,11 @@
 """
 Audit logging configuration for DefectDojo.
 
-This module handles conditional registration of models with either django-auditlog
-or django-pghistory based on the DD_AUDITLOG_TYPE setting.
+This module handles registration of models with django-pghistory.
+django-auditlog support has been removed.
 """
-import contextlib
 import logging
+import os
 import sys
 
 import pghistory
@@ -80,13 +80,6 @@ def _flush_models_in_batches(models_to_flush, timestamp_field: str, retention_pe
     return total_deleted, total_batches, reached_any_limit
 
 
-def _flush_django_auditlog(retention_period: int, batch_size: int, max_batches: int, *, dry_run: bool = False) -> tuple[int, int, bool]:
-    # Import inside to avoid model import issues at startup
-    from auditlog.models import LogEntry  # noqa: PLC0415
-
-    return _flush_models_in_batches([LogEntry], "timestamp", retention_period, batch_size, max_batches, dry_run=dry_run)
-
-
 def _iter_pghistory_event_models():
     """Yield pghistory Event models registered under the dojo app."""
     for model in apps.get_app_config("dojo").get_models():
@@ -107,8 +100,7 @@ def run_flush_auditlog(retention_period: int | None = None,
                        *,
                        dry_run: bool = False) -> tuple[int, int, bool]:
     """
-    Deletes audit entries older than the configured retention from both
-    django-auditlog and django-pghistory log entries.
+    Deletes audit entries older than the configured retention from django-pghistory log entries.
 
     Returns a tuple of (deleted_total, batches_done, reached_limit).
     """
@@ -121,93 +113,13 @@ def run_flush_auditlog(retention_period: int | None = None,
     max_batches = max_batches if max_batches is not None else getattr(settings, "AUDITLOG_FLUSH_MAX_BATCHES", 100)
 
     phase = "DRY RUN" if dry_run else "Cleanup"
-    logger.info("Running %s for django-auditlog entries with %d Months retention across all backends", phase, retention_period)
-    d_deleted, d_batches, d_limit = _flush_django_auditlog(retention_period, batch_size, max_batches, dry_run=dry_run)
     logger.info("Running %s for django-pghistory entries with %d Months retention across all backends", phase, retention_period)
     p_deleted, p_batches, p_limit = _flush_pghistory_events(retention_period, batch_size, max_batches, dry_run=dry_run)
 
-    total_deleted = d_deleted + p_deleted
-    total_batches = d_batches + p_batches
-    reached_limit = bool(d_limit or p_limit)
-
     verb = "would delete" if dry_run else "deleted"
-    logger.info("Audit flush summary: django-auditlog %s=%s batches=%s; pghistory %s=%s batches=%s; total_%s=%s total_batches=%s",
-                verb, d_deleted, d_batches, verb, p_deleted, p_batches, verb.replace(" ", "_"), total_deleted, total_batches)
-
-    return total_deleted, total_batches, reached_limit
-
-
-def enable_django_auditlog():
-    """Enable django-auditlog by registering models."""
-    # Import inside function to avoid AppRegistryNotReady errors
-    from auditlog.registry import auditlog  # noqa: PLC0415
-
-    from dojo.models import (  # noqa: PLC0415
-        Cred_User,
-        Dojo_User,
-        Endpoint,
-        Engagement,
-        Finding,
-        Finding_Group,
-        Finding_Template,
-        Notification_Webhooks,
-        Product,
-        Product_Type,
-        Risk_Acceptance,
-        Test,
-    )
-
-    logger.info("Enabling django-auditlog: Registering models")
-    auditlog.register(Dojo_User, exclude_fields=["password"])
-    auditlog.register(Endpoint)
-    auditlog.register(Engagement)
-    auditlog.register(Finding, m2m_fields={"reviewers"})
-    auditlog.register(Finding_Group)
-    auditlog.register(Product_Type)
-    auditlog.register(Product)
-    auditlog.register(Test)
-    auditlog.register(Risk_Acceptance)
-    auditlog.register(Finding_Template)
-    auditlog.register(Cred_User, exclude_fields=["password"])
-    auditlog.register(Notification_Webhooks, exclude_fields=["header_name", "header_value"])
-    logger.info("Successfully enabled django-auditlog")
-
-
-def disable_django_auditlog():
-    """Disable django-auditlog by unregistering models."""
-    # Import inside function to avoid AppRegistryNotReady errors
-    from auditlog.registry import auditlog  # noqa: PLC0415
-
-    from dojo.models import (  # noqa: PLC0415
-        Cred_User,
-        Dojo_User,
-        Endpoint,
-        Engagement,
-        Finding,
-        Finding_Group,
-        Finding_Template,
-        Notification_Webhooks,
-        Product,
-        Product_Type,
-        Risk_Acceptance,
-        Test,
-    )
+    logger.info("Audit flush summary: pghistory %s=%s batches=%s", verb, p_deleted, p_batches)
 
-    # Only log during actual application startup, not during shell commands
-    if "shell" not in sys.argv:
-        logger.info("Django-auditlog disabled - unregistering models")
-
-    # Unregister all models from auditlog
-    models_to_unregister = [
-        Dojo_User, Endpoint, Engagement, Finding, Finding_Group,
-        Product_Type, Product, Test, Risk_Acceptance, Finding_Template,
-        Cred_User, Notification_Webhooks,
-    ]
-
-    for model in models_to_unregister:
-        with contextlib.suppress(Exception):
-            # Model might not be registered, ignore the error
-            auditlog.unregister(model)
+    return p_deleted, p_batches, bool(p_limit)
 
 
 def register_django_pghistory_models():
@@ -308,6 +220,26 @@ def register_django_pghistory_models():
         },
     )(Finding)
 
+    # # Track the reviewers ManyToMany relationship through table
+    # # This tracks additions/removals of reviewers from findings
+    # reviewers_through = Finding._meta.get_field("reviewers").remote_field.through
+    # if reviewers_through:
+    #     logger.info(f"Tracking reviewers M2M through table: {reviewers_through} (db_table: {reviewers_through._meta.db_table})")
+    #     pghistory.track(
+    #         pghistory.InsertEvent(),
+    #         pghistory.DeleteEvent(),
+    #         meta={
+    #             "indexes": [
+    #                 models.Index(fields=["pgh_created_at"]),
+    #                 models.Index(fields=["pgh_label"]),
+    #                 models.Index(fields=["pgh_context_id"]),
+    #             ],
+    #         },
+    #     )(reviewers_through)
+    #     logger.info("Successfully registered pghistory tracking for reviewers through table")
+    # else:
+    #     logger.warning("Could not find reviewers through table for Finding model!")
+
     pghistory.track(
         pghistory.InsertEvent(),
         pghistory.UpdateEvent(condition=pghistory.AnyChange(exclude_auto=True)),
@@ -427,30 +359,6 @@ def register_django_pghistory_models():
         logger.info("Successfully registered models with django-pghistory")
 
 
-def enable_django_pghistory():
-    """Enable django-pghistory by enabling triggers."""
-    logger.info("Enabling django-pghistory: Enabling triggers")
-
-    # Enable pghistory triggers
-    try:
-        call_command("pgtrigger", "enable")
-        logger.info("Successfully enabled pghistory triggers")
-    except Exception as e:
-        logger.warning(f"Failed to enable pgtrigger triggers: {e}")
-        # Don't raise the exception as this shouldn't prevent Django from starting
-
-
-def disable_django_pghistory():
-    """Disable django-pghistory by disabling triggers."""
-    logger.info("Disabling django-pghistory: Disabling triggers")
-    try:
-        call_command("pgtrigger", "disable")
-        logger.info("Successfully disabled pghistory triggers")
-    except Exception as e:
-        logger.warning(f"Failed to disable pgtrigger triggers: {e}")
-        # Don't raise the exception as this shouldn't prevent Django from starting
-
-
 def configure_pghistory_triggers():
     """
     Configure pghistory triggers based on audit settings.
@@ -466,44 +374,52 @@ def configure_pghistory_triggers():
         except Exception as e:
             logger.error(f"Failed to disable pghistory triggers: {e}")
             raise
-    elif settings.AUDITLOG_TYPE == "django-pghistory":
+    else:
+        # Only pghistory is supported now
         try:
             call_command("pgtrigger", "enable")
             logger.info("Successfully enabled pghistory triggers")
         except Exception as e:
             logger.error(f"Failed to enable pghistory triggers: {e}")
             raise
-    else:
-        try:
-            call_command("pgtrigger", "disable")
-            logger.info("Successfully disabled pghistory triggers")
-        except Exception as e:
-            logger.error(f"Failed to disable pghistory triggers: {e}")
-            raise
 
 
 def configure_audit_system():
     """
     Configure the audit system based on settings.
 
-    Note: This function only handles auditlog registration. pghistory model registration
-    is handled in apps.py, and trigger management should be done via the
-    configure_pghistory_triggers() function to avoid database access during initialization.
+    django-auditlog is no longer supported. Only django-pghistory is allowed.
     """
     # Only log during actual application startup, not during shell commands
     log_enabled = "shell" not in sys.argv
 
+    # Fail if DD_AUDITLOG_TYPE is still configured (removed setting)
+    auditlog_type_env = os.environ.get("DD_AUDITLOG_TYPE")
+    if auditlog_type_env:
+        error_msg = (
+            "DD_AUDITLOG_TYPE environment variable is no longer supported. "
+            "DefectDojo now exclusively uses django-pghistory for audit logging. "
+            "Please remove DD_AUDITLOG_TYPE from your environment configuration. "
+            "All new audit entries will be created using django-pghistory automatically."
+        )
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
+    # Fail if AUDITLOG_TYPE is manually set in settings files (removed setting)
+    if hasattr(settings, "AUDITLOG_TYPE"):
+        error_msg = (
+            "AUDITLOG_TYPE setting is no longer supported. "
+            "DefectDojo now exclusively uses django-pghistory for audit logging. "
+            "Please remove AUDITLOG_TYPE from your settings file (settings.dist.py or local_settings.py). "
+            "All new audit entries will be created using django-pghistory automatically."
+        )
+        logger.error(error_msg)
+        raise ValueError(error_msg)
+
     if not settings.ENABLE_AUDITLOG:
         if log_enabled:
             logger.info("Audit logging disabled")
-        disable_django_auditlog()
         return
 
-    if settings.AUDITLOG_TYPE == "django-auditlog":
-        if log_enabled:
-            logger.info("Configuring audit system: django-auditlog enabled")
-        enable_django_auditlog()
-    else:
-        if log_enabled:
-            logger.info("django-auditlog disabled (pghistory or other audit type selected)")
-        disable_django_auditlog()
+    if log_enabled:
+        logger.info("Audit logging configured: django-pghistory")

dojo/management/commands/pghistory_backfill.py

@@ -140,11 +140,10 @@ def disable_db_logging(self):
         )
 
     def handle(self, *args, **options):
-        if not settings.ENABLE_AUDITLOG or settings.AUDITLOG_TYPE != "django-pghistory":
+        if not settings.ENABLE_AUDITLOG:
             self.stdout.write(
                 self.style.WARNING(
-                    "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True and "
-                    "DD_AUDITLOG_TYPE=django-pghistory",
+                    "pghistory is not enabled. Set DD_ENABLE_AUDITLOG=True",
                 ),
             )
             return