feat: PK-based device matching via metadata.source_match.netbox_id (#158)

* feat: PK-based device matching via metadata.source_match.netbox_id

When an ingested entity carries metadata.source_match.netbox_id, use
direct PK lookup instead of constraint-based matching. This supports
the Discovery use case where targets are known NetBox objects.

- Extract metadata from entity before _ensure_snake_case strips it
- In _resolve_existing_references, do PK lookup before matcher chain
- Raise ChangeSetException if netbox_id points to nonexistent object
- Invalid (non-numeric) netbox_id logs a warning and falls through
  to normal matching

* fix: remove unnecessary else after continue (ruff RET507)

Author: mfiedorowicz

netbox_diode_plugin/api/differ.py

@@ -195,6 +195,7 @@ def _generate_changeset(entity: dict, object_type: str) -> ChangeSetResult:
         object_type = entity.pop("_object_type")
         _ = entity.pop("_uuid")
         instance = entity.pop("_instance", None)
+        entity.pop("_netbox_id", None)
         _merge_warnings(warnings, object_type, entity.pop("_warnings", None))
         if instance:
             # the prior state is another new object...

netbox_diode_plugin/api/transformer.py

@@ -24,6 +24,7 @@
     CUSTOM_FIELD_OBJECT_REFERENCE_TYPE,
     apply_format_transformations,
     get_json_ref_info,
+    get_object_type_model,
     get_primary_value,
     legal_fields,
 )
@@ -127,6 +128,9 @@ def _transform_proto_json_1(proto_json: dict, object_type: str, supported_models
         "_warnings": {},
     }
 
+    # extract metadata before _ensure_snake_case strips it as an unknown field
+    metadata = dict.pop(proto_json, "metadata", None)
+
     # handle camelCase protoJSON if provided...
     proto_json = _ensure_snake_case(proto_json, object_type)
     apply_format_transformations(proto_json, object_type)
@@ -151,6 +155,17 @@ def _transform_proto_json_1(proto_json: dict, object_type: str, supported_models
         node['_refs'].update(custom_fields_refs)
         nodes += nested
 
+    # process extracted metadata for PK-based matching
+    if metadata and isinstance(metadata, dict):
+        source_match = metadata.get("source_match", {})
+        if isinstance(source_match, dict):
+            netbox_id_raw = source_match.get("netbox_id")
+            if netbox_id_raw is not None:
+                try:
+                    node['_netbox_id'] = int(netbox_id_raw)
+                except (ValueError, TypeError):
+                    node['_warnings']['metadata'] = [f"Invalid netbox_id: {netbox_id_raw}"]
+
     supported_fields = _supported_diode_fields(object_type, supported_models)
     def is_supported(field_name, ref_info):
         if ref_info is None:
@@ -532,6 +547,27 @@ def _resolve_existing_references(entities: list[dict]) -> list[dict]:
             resolved.append(data)
             continue
 
+        # PK-based lookup: if metadata provided a netbox_id, use it directly
+        netbox_id = data.pop('_netbox_id', None)
+        if netbox_id is not None:
+            model_class = get_object_type_model(object_type)
+            existing = model_class.objects.filter(pk=netbox_id).first()
+            if existing is not None:
+                fp = (object_type, existing.id)
+                if fp in seen:
+                    logger.warning(f"objects resolved to the same existing id after deduplication: {seen[fp]} and {data}")
+                else:
+                    seen[fp] = data
+                data['id'] = existing.id
+                data['_instance'] = existing
+                new_refs[data['_uuid']] = existing.id
+                resolved.append(data)
+                continue
+            raise ChangeSetException(
+                f"Object not found for {object_type} with netbox_id={netbox_id}",
+                errors={NON_FIELD_ERRORS: [f"No {object_type} found with id {netbox_id}"]}
+            )
+
         existing = find_existing_object(data, object_type)
         if existing is not None:
             fp = (object_type, existing.id)

netbox_diode_plugin/tests/v4.5.x/tests/test_api_generate_diff.py

@@ -9,7 +9,7 @@
 from uuid import uuid4
 
 from core.models import ObjectType
-from dcim.models import Manufacturer, RackType, Site
+from dcim.models import Device, DeviceRole, DeviceType, Interface, Manufacturer, RackType, Site
 from extras.models import CustomField
 from extras.models.customfields import CustomFieldTypeChoices
 from rest_framework import status
@@ -419,3 +419,246 @@ def send_request(self, payload, status_code=status.HTTP_200_OK):
         )
         self.assertEqual(response.status_code, status_code)
         return response
+
+
+class PKBasedMatchingTestCase(APITestCase):
+    """Test PK-based matching via metadata.source_match.netbox_id."""
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.url = "/netbox/api/plugins/diode/generate-diff/"
+
+        self.authorization_header = {"HTTP_AUTHORIZATION": "Bearer mocked_oauth_token"}
+        self.diode_user = SimpleNamespace(
+            user=get_diode_user(),
+            token_scopes=["netbox:read", "netbox:write"],
+            token_data={"scope": "netbox:read netbox:write"},
+        )
+
+        self.introspect_patcher = mock.patch.object(
+            DiodeOAuth2Authentication,
+            "_introspect_token",
+            return_value=self.diode_user,
+        )
+        self.introspect_patcher.start()
+
+        self.site = Site.objects.create(name="PK Test Site", slug="pk-test-site")
+        manufacturer = Manufacturer.objects.create(name="PK Test Manufacturer", slug="pk-test-manufacturer")
+        device_type = DeviceType.objects.create(
+            manufacturer=manufacturer, model="PK Test Type", slug="pk-test-type"
+        )
+        self.role = DeviceRole.objects.create(name="PK Test Role", slug="pk-test-role", color="ff0000")
+        self.device = Device.objects.create(
+            name="PK Test Device",
+            device_type=device_type,
+            role=self.role,
+            site=self.site,
+            serial="ORIG-SERIAL",
+        )
+        self.interface = Interface.objects.create(
+            device=self.device,
+            name="eth0",
+            type="virtual",
+        )
+
+    def tearDown(self):
+        """Clean up after tests."""
+        self.introspect_patcher.stop()
+        super().tearDown()
+
+    def send_request(self, payload, status_code=status.HTTP_200_OK):
+        """Post the payload to the url and return the response."""
+        response = self.client.post(
+            self.url, data=payload, format="json", **self.authorization_header
+        )
+        self.assertEqual(response.status_code, status_code)
+        return response
+
+    def test_pk_match_noop(self):
+        """PK match with identical data produces no changes."""
+        payload = {
+            "object_type": "dcim.device",
+            "entity": {
+                "device": {
+                    "name": "PK Test Device",
+                    "serial": "ORIG-SERIAL",
+                    "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                    "role": {"name": "PK Test Role"},
+                    "site": {"name": "PK Test Site"},
+                    "metadata": {
+                        "source_match": {"netbox_id": self.device.pk},
+                    },
+                },
+            },
+        }
+        response = self.send_request(payload)
+        cs = response.json().get("change_set", {})
+        changes = cs.get("changes", [])
+        # all changes should be noop since data matches
+        device_changes = [c for c in changes if c["object_type"] == "dcim.device"]
+        self.assertTrue(len(device_changes) <= 1)
+        if device_changes:
+            self.assertEqual(device_changes[0]["change_type"], "noop")
+
+    def test_pk_match_update(self):
+        """PK match with changed attribute produces update changeset."""
+        payload = {
+            "object_type": "dcim.device",
+            "entity": {
+                "device": {
+                    "name": "PK Test Device",
+                    "serial": "NEW-SERIAL",
+                    "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                    "role": {"name": "PK Test Role"},
+                    "site": {"name": "PK Test Site"},
+                    "metadata": {
+                        "source_match": {"netbox_id": self.device.pk},
+                    },
+                },
+            },
+        }
+        response = self.send_request(payload)
+        cs = response.json().get("change_set", {})
+        changes = cs.get("changes", [])
+        device_changes = [c for c in changes if c["object_type"] == "dcim.device"]
+        self.assertEqual(len(device_changes), 1)
+        change = device_changes[0]
+        self.assertEqual(change["change_type"], "update")
+        self.assertEqual(change["object_id"], self.device.pk)
+        self.assertEqual(change["data"]["serial"], "NEW-SERIAL")
+
+    def test_pk_match_ignores_name(self):
+        """PK match finds the device even when the name is different — produces update, not create."""
+        payload = {
+            "object_type": "dcim.device",
+            "entity": {
+                "device": {
+                    "name": "Completely Different Name",
+                    "serial": "ORIG-SERIAL",
+                    "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                    "role": {"name": "PK Test Role"},
+                    "site": {"name": "PK Test Site"},
+                    "metadata": {
+                        "source_match": {"netbox_id": self.device.pk},
+                    },
+                },
+            },
+        }
+        response = self.send_request(payload)
+        cs = response.json().get("change_set", {})
+        changes = cs.get("changes", [])
+        device_changes = [c for c in changes if c["object_type"] == "dcim.device"]
+        self.assertEqual(len(device_changes), 1)
+        change = device_changes[0]
+        self.assertEqual(change["change_type"], "update")
+        self.assertEqual(change["object_id"], self.device.pk)
+        # name change should be in the diff
+        self.assertEqual(change["data"]["name"], "Completely Different Name")
+
+    def test_pk_not_found_raises_error(self):
+        """PK that doesn't exist returns an error, not a create."""
+        payload = {
+            "object_type": "dcim.device",
+            "entity": {
+                "device": {
+                    "name": "Ghost Device",
+                    "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                    "role": {"name": "PK Test Role"},
+                    "site": {"name": "PK Test Site"},
+                    "metadata": {
+                        "source_match": {"netbox_id": 999999},
+                    },
+                },
+            },
+        }
+        response = self.send_request(payload, status_code=status.HTTP_400_BAD_REQUEST)
+        errors = response.json().get("errors", {})
+        self.assertTrue(len(errors) > 0)
+
+    def test_no_metadata_uses_normal_matching(self):
+        """Without metadata, normal constraint-based matching applies."""
+        payload = {
+            "object_type": "dcim.device",
+            "entity": {
+                "device": {
+                    "name": "PK Test Device",
+                    "serial": "CHANGED-SERIAL",
+                    "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                    "role": {"name": "PK Test Role"},
+                    "site": {"name": "PK Test Site"},
+                },
+            },
+        }
+        response = self.send_request(payload)
+        cs = response.json().get("change_set", {})
+        changes = cs.get("changes", [])
+        device_changes = [c for c in changes if c["object_type"] == "dcim.device"]
+        self.assertEqual(len(device_changes), 1)
+        change = device_changes[0]
+        # should still match by name+site and produce an update
+        self.assertEqual(change["change_type"], "update")
+        self.assertEqual(change["object_id"], self.device.pk)
+
+    def test_pk_match_device_via_nested_interface(self):
+        """Interface ingested with parent device carrying netbox_id — device matched by PK, interface by name."""
+        payload = {
+            "object_type": "dcim.interface",
+            "entity": {
+                "interface": {
+                    "name": "eth0",
+                    "type": "virtual",
+                    "mtu": 9000,
+                    "device": {
+                        "name": "Irrelevant Name",
+                        "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                        "role": {"name": "PK Test Role"},
+                        "site": {"name": "PK Test Site"},
+                        "metadata": {
+                            "source_match": {"netbox_id": self.device.pk},
+                        },
+                    },
+                },
+            },
+        }
+        response = self.send_request(payload)
+        cs = response.json().get("change_set", {})
+        changes = cs.get("changes", [])
+        device_changes = [c for c in changes if c["object_type"] == "dcim.device"]
+        interface_changes = [c for c in changes if c["object_type"] == "dcim.interface"]
+        # device should be matched by PK (name differs so it's an update)
+        self.assertEqual(len(device_changes), 1)
+        self.assertEqual(device_changes[0]["change_type"], "update")
+        self.assertEqual(device_changes[0]["object_id"], self.device.pk)
+        # interface should be matched by name within the PK-resolved device
+        self.assertEqual(len(interface_changes), 1)
+        self.assertEqual(interface_changes[0]["change_type"], "update")
+        self.assertEqual(interface_changes[0]["object_id"], self.interface.pk)
+
+    def test_invalid_netbox_id_falls_through(self):
+        """Invalid (non-numeric) netbox_id is ignored with a warning, falls through to normal matching."""
+        payload = {
+            "object_type": "dcim.device",
+            "entity": {
+                "device": {
+                    "name": "PK Test Device",
+                    "serial": "CHANGED-FOR-INVALID-PK-TEST",
+                    "device_type": {"model": "PK Test Type", "manufacturer": {"name": "PK Test Manufacturer"}},
+                    "role": {"name": "PK Test Role"},
+                    "site": {"name": "PK Test Site"},
+                    "metadata": {
+                        "source_match": {"netbox_id": "not-a-number"},
+                    },
+                },
+            },
+        }
+        response = self.send_request(payload)
+        cs = response.json().get("change_set", {})
+        changes = cs.get("changes", [])
+        device_changes = [c for c in changes if c["object_type"] == "dcim.device"]
+        self.assertEqual(len(device_changes), 1)
+        # should fall through to normal matching and find the device by name
+        self.assertEqual(device_changes[0]["change_type"], "update")
+        self.assertEqual(device_changes[0]["object_id"], self.device.pk)
+        # warning should be present
+        warnings = cs.get("warnings", {})
+        self.assertIn("metadata", warnings.get("dcim.device", {}))