Skip to content

Commit 40c7a07

Browse files
Maffoochclaude
andcommitted
perf(migrations): build vuln-id index + unique constraint concurrently (F9)
0280's lookup index and 0282's (finding, vulnerability_id) unique constraint were built with plain AddIndex / AddConstraint, taking a SHARE lock (blocks writes) and an ACCESS EXCLUSIVE lock (blocks reads+writes) for the full build over the whole dojo_vulnerability_id table -- a material downtime window on large instances. Build both CONCURRENTLY instead (atomic=False), keeping the work in-migration: - 0280: AddIndexConcurrently for dojo_vuln_id_lookup_idx. - 0282: CREATE UNIQUE INDEX CONCURRENTLY, then ADD CONSTRAINT ... USING INDEX (brief lock to adopt the pre-built index); SeparateDatabaseAndState keeps the UniqueConstraint in Django's model state. Verified on a 100k-finding / 174k-id dataset with duplicate rows: 0281 dedupe -> 0, constraint built valid as a real UNIQUE constraint, makemigrations --check clean. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
1 parent 7bd952b commit 40c7a07

2 files changed

Lines changed: 44 additions & 9 deletions

File tree

dojo/db_migrations/0280_vulnerability_id_type.py

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,22 @@
1+
from django.contrib.postgres.operations import AddIndexConcurrently
12
from django.db import migrations, models
23

34

45
class Migration(migrations.Migration):
56

6-
"""Schema only: add the autodetected vulnerability_id_type column and a leading index on
7-
vulnerability_id. The data backfill (0277) and the unique constraint (0278, which requires
7+
"""
8+
Schema only: add the autodetected vulnerability_id_type column and a leading index on
9+
vulnerability_id. The data backfill (0281) and the unique constraint (0282, which requires
810
the dedupe data step first) are kept in separate migrations so data migrations are never
9-
mixed with schema migrations."""
11+
mixed with schema migrations.
12+
13+
The lookup index is built with CREATE INDEX CONCURRENTLY (no SHARE lock, so writes to
14+
dojo_vulnerability_id keep flowing during the build), which requires a non-atomic migration.
15+
Adding the nullable, all-NULL vulnerability_id_type column and its own index is a fast catalog
16+
operation, so it stays a plain AddField.
17+
"""
18+
19+
atomic = False
1020

1121
dependencies = [
1222
("dojo", "0279_jira_project_transition_fields"),
@@ -18,7 +28,7 @@ class Migration(migrations.Migration):
1828
name="vulnerability_id_type",
1929
field=models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True),
2030
),
21-
migrations.AddIndex(
31+
AddIndexConcurrently(
2232
model_name="vulnerability_id",
2333
index=models.Index(fields=["vulnerability_id"], name="dojo_vuln_id_lookup_idx"),
2434
),

dojo/db_migrations/0282_unique_finding_vulnerability_id.py

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,41 @@
33

44
class Migration(migrations.Migration):
55

6-
"""Schema only: add the (finding, vulnerability_id) unique constraint. Runs after the 0277
7-
dedupe data step so the constraint can be created without violating existing duplicate rows."""
6+
"""
7+
Schema only: add the (finding, vulnerability_id) unique constraint. Runs after the 0281
8+
dedupe data step so the constraint can be created without violating existing duplicate rows.
9+
10+
The unique index is built with CREATE UNIQUE INDEX CONCURRENTLY (no ACCESS EXCLUSIVE
11+
full-build lock, so reads and writes keep flowing during the build) and then attached as the
12+
constraint with ADD CONSTRAINT ... USING INDEX, which only takes a brief lock to adopt the
13+
already-built index. Both steps run inside this migration; CONCURRENTLY requires atomic=False.
14+
"""
15+
16+
atomic = False
817

918
dependencies = [
1019
("dojo", "0281_backfill_vulnerability_id_type"),
1120
]
1221

1322
operations = [
14-
migrations.AddConstraint(
15-
model_name="vulnerability_id",
16-
constraint=models.UniqueConstraint(fields=("finding", "vulnerability_id"), name="unique_finding_vulnerability_id"),
23+
migrations.RunSQL(
24+
sql="CREATE UNIQUE INDEX CONCURRENTLY IF NOT EXISTS unique_finding_vulnerability_id "
25+
"ON dojo_vulnerability_id (finding_id, vulnerability_id);",
26+
reverse_sql="DROP INDEX CONCURRENTLY IF EXISTS unique_finding_vulnerability_id;",
27+
),
28+
migrations.SeparateDatabaseAndState(
29+
database_operations=[
30+
migrations.RunSQL(
31+
sql="ALTER TABLE dojo_vulnerability_id ADD CONSTRAINT unique_finding_vulnerability_id "
32+
"UNIQUE USING INDEX unique_finding_vulnerability_id;",
33+
reverse_sql="ALTER TABLE dojo_vulnerability_id DROP CONSTRAINT IF EXISTS unique_finding_vulnerability_id;",
34+
),
35+
],
36+
state_operations=[
37+
migrations.AddConstraint(
38+
model_name="vulnerability_id",
39+
constraint=models.UniqueConstraint(fields=("finding", "vulnerability_id"), name="unique_finding_vulnerability_id"),
40+
),
41+
],
1742
),
1843
]

0 commit comments

Comments
 (0)