Skip to content

Commit ca6d54a

Browse files
committed
perf(plugin): preload ContentType + tag IDs once per changeset
PR 2 of BULK-ORM Sprint 1. Adds `_preload_changeset_cache(change_set, request)` called at the top of `apply_changeset`. It walks the changeset once, collects every distinct object_type and every tag slug, then: - Calls `ContentType.objects.get_for_models(*models)` — a single query that warms Django's per-process content-type cache so subsequent per-change `get_for_model()` calls are free. - Issues one `Tag.objects.filter(slug__in=...).values_list('id','slug')` query and stashes `tag_ids_by_slug` on `request._diode_preload`. PR 4 will reuse `tag_ids_by_slug` to bulk-write `TaggedItem` rows. Test `test_preload_cache.py` asserts: - After preload, `get_for_model(Site)` runs in 0 queries. - Tag IDs for slugs referenced by non-NOOP changes are resolved in <= 2 queries (1 ContentType warm + 1 Tag fetch).
1 parent 8fd8dd4 commit ca6d54a

2 files changed

Lines changed: 135 additions & 0 deletions

File tree

netbox_diode_plugin/api/applier.py

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,11 @@
55

66
import logging
77

8+
from django.contrib.contenttypes.models import ContentType
89
from django.core.exceptions import ObjectDoesNotExist
910
from django.db import models, transaction
1011
from django.db.utils import IntegrityError
12+
from extras.models import Tag
1113
from rest_framework.exceptions import ValidationError as ValidationError
1214

1315
from .common import NON_FIELD_ERRORS, Change, ChangeSet, ChangeSetException, ChangeSetResult, ChangeType, error_from_validation_error
@@ -23,6 +25,7 @@
2325
def apply_changeset(change_set: ChangeSet, request) -> ChangeSetResult:
2426
"""Apply a change set."""
2527
_validate_change_set(change_set)
28+
_preload_changeset_cache(change_set, request)
2629

2730
created = {}
2831
for change in change_set.changes:
@@ -56,6 +59,63 @@ def apply_changeset(change_set: ChangeSet, request) -> ChangeSetResult:
5659
id=change_set.id,
5760
)
5861

62+
def _preload_changeset_cache(change_set: ChangeSet, request) -> dict:
63+
"""
64+
Warm Django's ContentType cache and prefetch tag IDs once per changeset.
65+
66+
Without this, every per-change `ContentType.objects.get_for_model(...)` and
67+
every `Tag.objects.get(slug=...)` issues its own SQL. Stashing the result on
68+
`request._diode_preload` lets later code paths (PR 4 bulk-tag write) reuse
69+
it without re-querying.
70+
"""
71+
models_to_warm: dict[str, models.Model] = {}
72+
tag_slugs: set[str] = set()
73+
74+
for change in change_set.changes:
75+
if change.change_type == ChangeType.NOOP:
76+
continue
77+
ot = change.object_type
78+
if ot and ot not in models_to_warm:
79+
try:
80+
models_to_warm[ot] = get_object_type_model(ot)
81+
except Exception:
82+
# Unknown model — let the main apply path raise the proper error.
83+
continue
84+
for slug in _iter_tag_slugs(change):
85+
tag_slugs.add(slug)
86+
87+
if models_to_warm:
88+
# Populates Django's per-process ContentType cache in a single query.
89+
ContentType.objects.get_for_models(*models_to_warm.values())
90+
91+
tag_ids_by_slug: dict[str, int] = {}
92+
if tag_slugs:
93+
for tag_id, slug in Tag.objects.filter(slug__in=tag_slugs).values_list("id", "slug"):
94+
tag_ids_by_slug[slug] = tag_id
95+
96+
preload = {
97+
"tag_ids_by_slug": tag_ids_by_slug,
98+
"models_by_object_type": models_to_warm,
99+
}
100+
if request is not None:
101+
request._diode_preload = preload
102+
return preload
103+
104+
105+
def _iter_tag_slugs(change: Change):
106+
"""Yield string tag slugs from a change.data['tags'] list."""
107+
if not change.data:
108+
return
109+
tags = change.data.get("tags")
110+
if not isinstance(tags, list):
111+
return
112+
for t in tags:
113+
if isinstance(t, str):
114+
yield t
115+
elif isinstance(t, dict) and isinstance(t.get("slug"), str):
116+
yield t["slug"]
117+
118+
59119
def _is_auto_created_component(object_type: str) -> bool:
60120
"""Check if the object type is auto-created from templates."""
61121
auto_created_components = [
Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
#!/usr/bin/env python
2+
# Copyright 2026 NetBox Labs, Inc.
3+
"""Diode NetBox Plugin - PR 2 preload cache tests."""
4+
5+
from types import SimpleNamespace
6+
7+
from dcim.models import Site
8+
from django.contrib.contenttypes.models import ContentType
9+
from django.db import connection
10+
from django.test import TestCase
11+
from django.test.utils import CaptureQueriesContext
12+
from extras.models import Tag
13+
14+
from netbox_diode_plugin.api.applier import _preload_changeset_cache
15+
from netbox_diode_plugin.api.common import Change, ChangeSet, ChangeType
16+
17+
18+
class PreloadChangesetCacheTestCase(TestCase):
19+
"""Verifies _preload_changeset_cache (PR 2)."""
20+
21+
def setUp(self):
22+
"""Create two tags + a 3-change changeset (one NOOP) for the assertions below."""
23+
self.tag_a = Tag.objects.create(name="alpha", slug="alpha")
24+
self.tag_b = Tag.objects.create(name="beta", slug="beta")
25+
26+
self.change_set = ChangeSet(
27+
id="11111111-1111-1111-1111-111111111111",
28+
changes=[
29+
Change(
30+
change_type=ChangeType.CREATE.value,
31+
object_type="dcim.site",
32+
ref_id="r1",
33+
data={"name": "S1", "slug": "s1", "tags": ["alpha", "beta"]},
34+
),
35+
Change(
36+
change_type=ChangeType.CREATE.value,
37+
object_type="dcim.site",
38+
ref_id="r2",
39+
# repeated tag — must dedup
40+
data={"name": "S2", "slug": "s2", "tags": ["alpha"]},
41+
),
42+
Change(
43+
change_type=ChangeType.NOOP.value,
44+
object_type="dcim.site",
45+
ref_id="r3",
46+
data={"tags": ["ignored"]},
47+
),
48+
],
49+
)
50+
51+
def test_contenttype_cache_hit_after_preload(self):
52+
"""get_for_model on a preloaded model issues 0 queries."""
53+
ContentType.objects.clear_cache()
54+
request = SimpleNamespace()
55+
_preload_changeset_cache(self.change_set, request)
56+
57+
with CaptureQueriesContext(connection) as ctx:
58+
ContentType.objects.get_for_model(Site)
59+
assert len(ctx.captured_queries) == 0, ctx.captured_queries
60+
61+
def test_tag_ids_collected_in_at_most_two_queries(self):
62+
"""Preload should resolve tag slugs in <= 2 queries total (1 ContentType + 1 Tag)."""
63+
ContentType.objects.clear_cache()
64+
request = SimpleNamespace()
65+
with CaptureQueriesContext(connection) as ctx:
66+
preload = _preload_changeset_cache(self.change_set, request)
67+
# NOOP changes are skipped, so "ignored" tag should NOT be loaded.
68+
assert preload["tag_ids_by_slug"] == {
69+
"alpha": self.tag_a.id,
70+
"beta": self.tag_b.id,
71+
}
72+
# 1 query for ContentType warm-up + 1 query for Tag.filter(...). Allow
73+
# a small slack in case Django prefers to issue a savepoint.
74+
assert len(ctx.captured_queries) <= 2, ctx.captured_queries
75+
assert request._diode_preload is preload

0 commit comments

Comments
 (0)