measure-evaluate: bundle performance indexes for production-scale installs

Author: Alexandra Pavlyshina

aidbox-custom-operations/measure-evaluate/setup.py

@@ -174,18 +174,16 @@ def execute_sql_file(filepath, label):
     if not os.path.exists(filepath):
         print(f"  SKIP {label} — not found")
         return
-
     with open(filepath) as f:
         content = f.read()
-
-    statements = [s.strip() for s in content.split(";") if s.strip()]
-    for stmt in statements:
-        try:
-            run_sql(stmt)
-        except Exception as e:
-            print(f"  FAIL {label}: {str(e)[:80]}")
-            return
-    print(f"  OK {label} — {len(statements)} statements")
+    try:
+        # Aidbox $sql wraps the whole call in one transaction — a mid-file
+        # error rolls back every preceding statement, so this is safe to retry.
+        run_sql(content)
+        # Rough count of '`;`' for the log line — not used for execution, just diagnostics.
+        print(f"  OK {label} — {content.count(';')} SQL statements")
+    except Exception as e:
+        print(f"  FAIL {label}: {str(e)[:200]}")
 
 
 def main():
@@ -230,11 +228,12 @@ def main():
                 print("  FAIL — Aidbox not responding. Is it running and activated?")
                 sys.exit(1)
 
-    # Create shared views + concepts table + shared exclusion functions
+    # Create shared views + concepts table + shared exclusion functions + indexes
     print("[2/5] Creating shared SQL infrastructure...")
     execute_sql_file(os.path.join(SCRIPT_DIR, "sql", "00-terminology.sql"), "Concepts table")
     execute_sql_file(os.path.join(SCRIPT_DIR, "sql", "01-views.sql"), "Shared views")
     execute_sql_file(os.path.join(SCRIPT_DIR, "sql", "02-shared-exclusions.sql"), "Shared exclusions")
+    execute_sql_file(os.path.join(SCRIPT_DIR, "sql", "03-performance.sql"), "Performance indexes")
 
     # Create stubs
     print("[3/5] Creating stub resources...")

aidbox-custom-operations/measure-evaluate/sql/03-performance.sql

@@ -0,0 +1,76 @@
+-- Performance Optimizations for Measure SQL
+-- Run AFTER 01-views.sql and 02-shared-exclusions.sql
+-- Safe to re-run (all CREATE INDEX IF NOT EXISTS, ANALYZE is idempotent)
+--
+-- Contains:
+--   1. Btree indexes on FHIR resource JSONB paths (speeds up flat view scans)
+--   2. ANALYZE to update planner statistics
+--
+-- Note on date-range indexes:
+--   We do NOT index timestamp-cast expressions like ((resource->'period'->>'start')::timestamptz),
+--   because Postgres rejects them ("functions in index expression must be marked IMMUTABLE":
+--   text->timestamp casts depend on the DateStyle GUC, hence STABLE not IMMUTABLE).
+--   In practice this does not matter for single-patient or small-cohort evaluation:
+--   once the subject indexes below limit the row set to one patient, an in-memory
+--   filter on dates is cheap. Date-range scans on the full table remain unindexed —
+--   for population-level evaluation at very large scale, plan a follow-up pass with
+--   to_timestamp(text, 'YYYY-MM-DD"T"HH24:MI:SS') or a maintained timestamp column.
+
+-- ============================================================
+-- 1. Indexes on FHIR resource tables
+-- These speed up the flat views by allowing index scans on JSONB paths.
+-- Named with ix_ prefix to distinguish from Aidbox's own indexes.
+-- ============================================================
+
+-- Patient: birthDate (age calculations), gender
+CREATE INDEX IF NOT EXISTS ix_patient_birthdate ON patient ((resource->>'birthDate'));
+CREATE INDEX IF NOT EXISTS ix_patient_gender ON patient ((resource->>'gender'));
+
+-- status columns are deliberately NOT indexed across all resources: the column
+-- has 3–5 distinct values in real data with one value (finished / final /
+-- completed / active) covering 90 %+ of rows, so the planner will prefer a
+-- sequential scan over a low-selectivity btree.
+
+-- Encounter: subject, type coding
+CREATE INDEX IF NOT EXISTS ix_encounter_subject ON encounter ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_encounter_type_code ON encounter ((resource->'type'->0->'coding'->0->>'system'), (resource->'type'->0->'coding'->0->>'code'));
+
+-- Observation: subject, code, value code
+CREATE INDEX IF NOT EXISTS ix_observation_subject ON observation ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_observation_code ON observation ((resource->'code'->'coding'->0->>'system'), (resource->'code'->'coding'->0->>'code'));
+CREATE INDEX IF NOT EXISTS ix_observation_value_code ON observation ((resource->'value'->'CodeableConcept'->'coding'->0->>'code')) WHERE resource->'value'->'CodeableConcept' IS NOT NULL;
+
+-- Condition: subject, code
+CREATE INDEX IF NOT EXISTS ix_condition_subject ON condition ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_condition_code ON condition ((resource->'code'->'coding'->0->>'system'), (resource->'code'->'coding'->0->>'code'));
+
+-- Procedure: subject, code
+CREATE INDEX IF NOT EXISTS ix_procedure_subject ON procedure ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_procedure_code ON procedure ((resource->'code'->'coding'->0->>'system'), (resource->'code'->'coding'->0->>'code'));
+
+-- ServiceRequest: subject, code
+CREATE INDEX IF NOT EXISTS ix_servicerequest_subject ON servicerequest ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_servicerequest_code ON servicerequest ((resource->'code'->'coding'->0->>'system'), (resource->'code'->'coding'->0->>'code'));
+
+-- MedicationRequest: subject, medication code
+CREATE INDEX IF NOT EXISTS ix_medrq_subject ON medicationrequest ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_medrq_med_code ON medicationrequest ((resource->'medication'->'CodeableConcept'->'coding'->0->>'system'), (resource->'medication'->'CodeableConcept'->'coding'->0->>'code'));
+
+-- DeviceRequest: subject, code
+CREATE INDEX IF NOT EXISTS ix_devicerq_subject ON devicerequest ((resource->'subject'->>'id'));
+CREATE INDEX IF NOT EXISTS ix_devicerq_code ON devicerequest ((resource->'code'->'CodeableConcept'->'coding'->0->>'system'), (resource->'code'->'CodeableConcept'->'coding'->0->>'code'));
+
+-- ============================================================
+-- 2. Update planner statistics
+-- On a fresh sample these complete in seconds. On a production-scale install
+-- with millions of rows each ANALYZE may take minutes — budget accordingly.
+-- ============================================================
+ANALYZE concepts;
+ANALYZE patient;
+ANALYZE encounter;
+ANALYZE observation;
+ANALYZE condition;
+ANALYZE procedure;
+ANALYZE servicerequest;
+ANALYZE medicationrequest;
+ANALYZE devicerequest;