🚚 release (#178)

Author: mfiedorowicz

netbox_diode_plugin/__init__.py

@@ -78,10 +78,25 @@ class NetBoxDiodePluginConfig(PluginConfig):
         # fields (empty-QuerySet truthiness bug), so under load every
         # instance.clean()/save() re-queries extras_customfield. Cache
         # is invalidated on CustomField post_save/post_delete signals.
+        #
+        # apply_buffer_change_logging: keep the audit trail intact but
+        # cut the per-save change-logging cost. During apply,
+        # ObjectChange serialisation skips the per-m2m-relation SELECTs
+        # that dominate `to_objectchange` (one query per m2m field on
+        # every save), and the rows are collected in an in-memory
+        # buffer instead of being written one at a time. On successful
+        # commit the buffer is flushed as a single `bulk_create`, with
+        # all objects' m2m relations resolved in one query per relation,
+        # and `post_save` is re-emitted so receivers connected to
+        # `post_save(ObjectChange)` still fire. The flush runs inline at
+        # commit, so the audit log stays immediately consistent.
+        # Mutually exclusive in intent with `apply_bypass_change_logging`
+        # - if both are enabled, bypass wins (no rows produced at all).
         "apply_bypass_counter_updates": False,
         "apply_bypass_change_logging": False,
         "apply_bypass_search_indexing": False,
         "apply_bypass_customfield_query_cache": False,
+        "apply_buffer_change_logging": False,
 
         # Per-entity retry on Postgres deadlock (SQLSTATE 40P01) during
         # the apply phase of /bulk-plan-apply. Cross-batch concurrent

netbox_diode_plugin/api/applier.py

@@ -10,8 +10,9 @@
 from django.db.utils import IntegrityError
 from rest_framework.exceptions import ValidationError as ValidationError
 
+from .change_log_buffer import snapshot_for_apply
 from .common import NON_FIELD_ERRORS, Change, ChangeSet, ChangeSetException, ChangeSetResult, ChangeType, error_from_validation_error
-from .matcher import find_existing_object, invalidate_find_obj_entry
+from .matcher import find_existing_object, invalidate_find_obj_entry, requires_pre_save_match
 from .plugin_utils import get_object_type_model, legal_fields
 from .profile import profiled
 from .supported_models import get_serializer_for_model
@@ -78,6 +79,7 @@ def _try_find_and_update_existing_instance(data: dict, object_type: str, seriali
     try:
         instance = find_existing_object(data, object_type)
         if instance:
+            snapshot_for_apply(instance)
             serializer = serializer_class(instance, data=data, partial=True, context={"request": request})
             serializer.is_valid(raise_exception=True)
             result = serializer.save()
@@ -110,8 +112,13 @@ def _apply_change(data: dict, model_class: models.Model, change: Change, created
         # For component types that may be auto-created from e.g. DeviceType or ModuleType templates,
         # try to find existing object first before attempting to create.
         # This prevents duplicates when components are instantiated during Device/Module save()
+        # The same find-first path also handles types whose logical match
+        # criteria are not enforced by a DB unique constraint (see
+        # matcher._REQUIRES_PRE_SAVE_MATCH): concurrent planners would
+        # otherwise each emit CREATE for the same logical row and both
+        # inserts would succeed without IntegrityError to fall back on.
         instance = None
-        if _is_auto_created_component(change.object_type):
+        if _is_auto_created_component(change.object_type) or requires_pre_save_match(change.object_type):
             instance = _try_find_and_update_existing_instance(data, change.object_type, serializer_class, request)
 
         if not instance:
@@ -124,6 +131,7 @@ def _apply_change(data: dict, model_class: models.Model, change: Change, created
     elif change_type == ChangeType.UPDATE:
         if object_id := change.object_id:
             instance = model_class.objects.get(id=object_id)
+            snapshot_for_apply(instance)
             serializer = serializer_class(instance, data=data, partial=True, context={"request": request})
             serializer.is_valid(raise_exception=True)
             serializer.save()