feat(data): detect stale imported reference data

Version imported reference-data semantics independently from the database schema and feed reader-version mismatches through the existing data drift prompt.\n\nCloses #162

Author: ApocDev

app/e2e/mut/data-format-drift.e2e.ts

@@ -0,0 +1,39 @@
+import { DatabaseSync } from "node:sqlite";
+import { expect, test } from "@playwright/test";
+import { activeProjectDbFile, goto } from "./helpers";
+
+test("an older imported-data format reuses the shared drift prompt", async ({ page }) => {
+  const file = activeProjectDbFile();
+  const db = new DatabaseSync(file);
+  const original = db
+    .prepare("SELECT value FROM meta WHERE key = 'data_format_version'")
+    .get() as { value: string | null } | undefined;
+  db.prepare(
+    "INSERT INTO meta (key, value) VALUES ('data_format_version', '0') ON CONFLICT(key) DO UPDATE SET value = '0'",
+  ).run();
+  db.close();
+
+  try {
+    await goto(page, "/");
+    const dialog = page.getByRole("dialog", { name: "Reference data is out of date" });
+    await expect(dialog).toBeVisible();
+    await expect(dialog).toContainText("PyOps now reads the Factorio dump differently");
+    await expect(dialog).toContainText(/Imported data format: v0 · Current reader: v\d+/);
+
+    await dialog.getByRole("button", { name: "Ignore for now" }).click();
+    await expect(dialog).toBeHidden();
+    await expect(page.locator("nav").getByRole("button", { name: "Data stale" })).toBeVisible();
+  } finally {
+    const restore = new DatabaseSync(file);
+    if (original) {
+      restore
+        .prepare(
+          "INSERT INTO meta (key, value) VALUES ('data_format_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value",
+        )
+        .run(original.value);
+    } else {
+      restore.prepare("DELETE FROM meta WHERE key = 'data_format_version'").run();
+    }
+    restore.close();
+  }
+});

app/src/components/app-nav.tsx

@@ -14,14 +14,14 @@ import { driftModal } from "../lib/drift-store";
 import { Button } from "#/components/ui/button.tsx";
 import { Tooltip } from "#/components/ui/tooltip.tsx";
 
-/** Persistent re-entry point for the data-sync modal: a small warning-toned chip in the
- * nav whenever the game's mods have drifted from the project's reference data
- * (so dismissing the popup doesn't strand it). Hidden when data is in sync. */
+/** Persistent re-entry point for the data-sync modal: a small warning-toned chip
+ * whenever the game's mods or PyOps' data reader have drifted from the project's
+ * reference data. Hidden when data is in sync. */
 function DataDriftIndicator() {
   const drift = useQuery({ queryKey: ["modDrift"], queryFn: () => modDriftFn() });
   if (!drift.data?.needsRedump) return null;
   return (
-    <Tooltip content="The game's mods changed since your last data sync — click to review and re-sync.">
+    <Tooltip content="Reference data no longer matches the current mods or data reader — click to review and re-sync.">
       <Button
         variant="ghost"
         onClick={() => driftModal.open()}

app/src/components/drift-modal.tsx

@@ -49,10 +49,10 @@ const writeDismissed = (sig: string) => {
   }
 };
 
-/** The guided data-sync experience: a themed modal that pops when mod drift is
- * detected (or is opened from the nav / Settings), lets you ignore or re-sync, and
- * then walks the dump as a step-by-step progress flow ending in a summary. Mounted
- * once in the root; it owns the periodic drift check and the sync polling. */
+/** The guided data-sync experience: a themed modal that pops when mod or importer
+ * drift is detected (or is opened from the nav / Settings), lets you ignore or
+ * re-sync, and then walks the dump as a step-by-step progress flow ending in a
+ * summary. Mounted once in the root; it owns the periodic drift check and polling. */
 export function DriftModal() {
   const qc = useQueryClient();
   const isOpen = useDriftModalOpen();
@@ -97,8 +97,12 @@ export function DriftModal() {
     prevConnected.current = connected;
   }, [connected, qc]);
 
-  // Auto-open once per drift signature (until it's dismissed or resolved by a sync).
-  const sig = drift.data?.needsRedump ? JSON.stringify(drift.data.drift) : null;
+  // Auto-open once per combined drift signature (until dismissed or resolved).
+  // Include the reader versions so each future format bump prompts exactly once,
+  // even when the mod set itself has not changed.
+  const sig = drift.data?.needsRedump
+    ? JSON.stringify({ mods: drift.data.drift, dataFormat: drift.data.dataFormat })
+    : null;
   useEffect(() => {
     if (sig && readDismissed() !== sig) driftModal.open();
   }, [sig]);
@@ -123,6 +127,8 @@ export function DriftModal() {
   if (!isOpen) return null;
 
   const hasDrift = !!drift.data?.needsRedump;
+  const hasModDrift = !!drift.data?.modsChanged;
+  const hasDataFormatDrift = !!drift.data?.dataFormat.stale;
   const gameUp = gameRunning.data?.running === true; // can't sync while the game runs
   const running = RUNNING.has(phase);
   const view: "prompt" | "running" | "done" | "error" = running
@@ -170,8 +176,8 @@ export function DriftModal() {
           <HelpButton title="Data drift" className="mr-7 ml-auto">
             <p>
               PyOps plans against a snapshot of the game&apos;s prototype data (recipes, items,
-              machines, techs). When the game&apos;s mod set changes, that snapshot drifts out of
-              date and this dialog offers a re-sync.
+              machines, techs). When the game&apos;s mod set or PyOps&apos; data reader changes,
+              that snapshot drifts out of date and this dialog offers a re-sync.
             </p>
             <p>
               A re-sync launches a headless copy of Factorio to re-dump the data, then imports it.
@@ -189,7 +195,10 @@ export function DriftModal() {
           {view === "prompt" && (
             <PromptBody
               hasDrift={hasDrift}
+              hasModDrift={hasModDrift}
+              hasDataFormatDrift={hasDataFormatDrift}
               drift={drift.data?.drift ?? null}
+              dataFormat={drift.data?.dataFormat}
               icons={icons}
               setIcons={setIcons}
               gameUp={gameUp}
@@ -272,13 +281,19 @@ const TITLES: Record<"running" | "done" | "error", TitleSpec> = {
 
 function PromptBody({
   hasDrift,
+  hasModDrift,
+  hasDataFormatDrift,
   drift,
+  dataFormat,
   icons,
   setIcons,
   gameUp,
 }: {
   hasDrift: boolean;
+  hasModDrift: boolean;
+  hasDataFormatDrift: boolean;
   drift: Parameters<typeof DriftChanges>[0]["drift"];
+  dataFormat?: { current: number; imported: number | null; stale: boolean };
   icons: boolean;
   setIcons: (v: boolean) => void;
   gameUp: boolean;
@@ -292,11 +307,24 @@ function PromptBody({
         </Callout>
       )}
       <p className="text-sm text-muted-foreground">
-        {hasDrift
-          ? "The game's mods changed since your last sync — re-sync to plan against the current data."
-          : "Re-dump the game's prototype data from the current mods and import it."}
+        {hasModDrift && hasDataFormatDrift
+          ? "The game's mods and PyOps' data reader changed since your last sync. Re-sync to rebuild this project's reference data."
+          : hasModDrift
+            ? "The game's mods changed since your last sync — re-sync to plan against the current data."
+            : hasDataFormatDrift
+              ? "PyOps now reads the Factorio dump differently than the version that built this project's reference data. Re-sync to rebuild it with the current reader."
+              : hasDrift
+                ? "This project's reference data needs to be rebuilt before planning continues."
+                : "Re-dump the game's prototype data from the current mods and import it."}
       </p>
-      {hasDrift && (
+      {hasDataFormatDrift && dataFormat && (
+        <div className="border border-warning/30 bg-warning/10 p-2 text-sm text-warning">
+          Imported data format:{" "}
+          {dataFormat.imported == null ? "unversioned" : `v${dataFormat.imported}`}
+          {" · "}Current reader: v{dataFormat.current}
+        </div>
+      )}
+      {hasModDrift && drift && (
         <div className="max-h-48 overflow-y-auto border border-border bg-muted/20 p-2">
           <DriftChanges drift={drift} />
         </div>

app/src/db/import-factorio.test.ts

@@ -3,6 +3,10 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import Database from "better-sqlite3";
 import { afterEach, beforeEach, describe, expect, it } from "vite-plus/test";
+import {
+  REFERENCE_DATA_FORMAT_META_KEY,
+  REFERENCE_DATA_FORMAT_VERSION,
+} from "../lib/data-format.ts";
 import { importFactorioDump } from "./import-factorio.ts";
 import { type TestDb, makeTestDb } from "./test-helpers.ts";
 
@@ -90,6 +94,18 @@ describe("importFactorioDump — Factorio 2.1 product probability", () => {
   });
 });
 
+describe("importFactorioDump — data format version", () => {
+  it("records the reader version after a successful two-pass import", () => {
+    const db = runImport({ recipe: {} });
+    const stored = db
+      .prepare("SELECT value FROM meta WHERE key = ?")
+      .pluck()
+      .get(REFERENCE_DATA_FORMAT_META_KEY);
+    expect(stored).toBe(String(REFERENCE_DATA_FORMAT_VERSION));
+    db.close();
+  });
+});
+
 describe("importFactorioDump — machine footprints", () => {
   it("derives placed tile dimensions from array and named selection boxes", () => {
     const db = runImport({

app/src/db/import-factorio.ts

@@ -15,6 +15,10 @@ import { homedir } from "node:os";
 import { dirname, join } from "node:path";
 import { synthesizePass2 } from "./synthesize.ts";
 import { temperatureFedDrain, type TempFedFluid } from "./fluid-energy.ts";
+import {
+  REFERENCE_DATA_FORMAT_META_KEY,
+  REFERENCE_DATA_FORMAT_VERSION,
+} from "../lib/data-format.ts";
 import { PROJECTS_DIR } from "../server/paths.server.ts";
 import { configureSqliteConnection } from "../server/provision.ts";
 
@@ -177,6 +181,7 @@ export type ImportSummary = {
   dump: string;
   dbUrl: string;
   ms: number;
+  dataFormatVersion: number;
   counts: Record<string, number>;
 };
 
@@ -623,8 +628,17 @@ export function importFactorioDump(
       null,
     parseSI,
   });
+  // Record compatibility only after both importer passes complete. A failed
+  // upgrade therefore keeps the previous version stale and prompts another sync.
+  ins.meta.run(REFERENCE_DATA_FORMAT_META_KEY, String(REFERENCE_DATA_FORMAT_VERSION));
   const count = (t: string) => (db.prepare(`SELECT count(*) c FROM ${t}`).get() as { c: number }).c;
   const counts = Object.fromEntries(COUNT_TABLES.map((t) => [t, count(t)]));
   db.close();
-  return { dump: DUMP, dbUrl: DB_URL, ms: Date.now() - t0, counts: { ...counts, ...synthetic } };
+  return {
+    dump: DUMP,
+    dbUrl: DB_URL,
+    ms: Date.now() - t0,
+    dataFormatVersion: REFERENCE_DATA_FORMAT_VERSION,
+    counts: { ...counts, ...synthetic },
+  };
 }

app/src/lib/data-format.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "vite-plus/test";
+import { REFERENCE_DATA_FORMAT_VERSION, referenceDataFormatStatus } from "./data-format.ts";
+
+describe("referenceDataFormatStatus", () => {
+  it("accepts reference data imported by the current reader", () => {
+    expect(referenceDataFormatStatus(String(REFERENCE_DATA_FORMAT_VERSION), true)).toEqual({
+      current: REFERENCE_DATA_FORMAT_VERSION,
+      imported: REFERENCE_DATA_FORMAT_VERSION,
+      stale: false,
+    });
+  });
+
+  it("marks unversioned, older, newer, and malformed imports stale", () => {
+    expect(referenceDataFormatStatus(undefined, true).stale).toBe(true);
+    expect(referenceDataFormatStatus("0", true).stale).toBe(true);
+    expect(referenceDataFormatStatus(String(REFERENCE_DATA_FORMAT_VERSION + 1), true).stale).toBe(
+      true,
+    );
+    expect(referenceDataFormatStatus("not-a-version", true).stale).toBe(true);
+  });
+
+  it("does not nag an empty project that already needs its initial sync", () => {
+    expect(referenceDataFormatStatus(undefined, false).stale).toBe(false);
+  });
+});

app/src/lib/data-format.ts

@@ -0,0 +1,31 @@
+/**
+ * Version of PyOps' normalized Factorio reference-data interpretation.
+ *
+ * Bump this whenever an importer or synthesis change means an existing project's
+ * reference tables must be rebuilt from the game dump. This is deliberately
+ * separate from SQLite migrations: migrations describe storage shape, while
+ * this version describes the meaning of imported rows.
+ */
+export const REFERENCE_DATA_FORMAT_VERSION = 1;
+export const REFERENCE_DATA_FORMAT_META_KEY = "data_format_version";
+
+export type ReferenceDataFormatStatus = {
+  current: number;
+  imported: number | null;
+  stale: boolean;
+};
+
+/** Missing versions represent imports made before format tracking existed. Empty
+ * projects are setup cases, not stale imports, so they do not trigger drift. */
+export function referenceDataFormatStatus(
+  stored: string | null | undefined,
+  hasReferenceData: boolean,
+): ReferenceDataFormatStatus {
+  const parsed = stored == null ? Number.NaN : Number(stored);
+  const imported = Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null;
+  return {
+    current: REFERENCE_DATA_FORMAT_VERSION,
+    imported,
+    stale: hasReferenceData && imported !== REFERENCE_DATA_FORMAT_VERSION,
+  };
+}

app/src/routes/settings.tsx

@@ -151,8 +151,8 @@ function GameDataTab() {
         <CardHeader className="justify-between">
           <CardTitle>Reference data</CardTitle>
           {drift.data?.needsRedump && (
-            <Tooltip content="The game's enabled mods or their versions changed since the last sync">
-              <Badge variant="destructive">Stale — mods changed</Badge>
+            <Tooltip content="The game's mods or PyOps' data reader changed since the last sync">
+              <Badge variant="destructive">Stale — re-sync needed</Badge>
             </Tooltip>
           )}
         </CardHeader>
@@ -173,6 +173,17 @@ function GameDataTab() {
                 {status.data.currentFingerprint &&
                   ` · Current mod list ${status.data.currentFingerprint}`}
               </div>
+              {drift.data?.dataFormat && (
+                <div
+                  className={`text-xs ${drift.data.dataFormat.stale ? "text-warning" : "text-muted-foreground"}`}
+                >
+                  Imported data format{" "}
+                  {drift.data.dataFormat.imported == null
+                    ? "unversioned"
+                    : `v${drift.data.dataFormat.imported}`}
+                  {" · "}Current reader v{drift.data.dataFormat.current}
+                </div>
+              )}
             </>
           )}
           <Button onClick={() => driftModal.open()} className="mt-1">
@@ -295,7 +306,7 @@ function ModDriftCard({ data }: { data: Awaited<ReturnType<typeof modDriftFn>> |
     );
   }
   const d = data.drift;
-  if (!data.needsRedump) {
+  if (!data.modsChanged || !d) {
     return (
       <Card>
         <CardHeader className="justify-between">

app/src/server/dump.server.ts

@@ -599,7 +599,7 @@ export function startDataSync(opts: { icons?: boolean } = {}): SyncState {
       step("import", "importing dump into sqlite");
       const summary = importFactorioDump({ dbUrl: currentDatabaseFile() });
       state.log.push(
-        `imported ${summary.counts.recipes} recipes / ${summary.counts.items} items in ${summary.ms}ms`,
+        `imported ${summary.counts.recipes} recipes / ${summary.counts.items} items with data reader v${summary.dataFormatVersion} in ${summary.ms}ms`,
       );
       if (icons) {
         step("atlas", "rebuilding icon atlas");

app/src/server/factorio.ts

@@ -11,6 +11,7 @@ import {
 } from "../lib/goals";
 import { extractRecipeToBlockDocs, withRecipeSet } from "../lib/block-doc";
 import type { FactoryFlowQualifier } from "../lib/factory-flow.ts";
+import { referenceDataFormatStatus } from "../lib/data-format.ts";
 
 /**
  * Server functions exposing the query layer to the client. Server-only modules
@@ -1093,14 +1094,16 @@ export const dataStatusFn = createServerFn({ method: "GET" }).handler(async () =
   };
 });
 
-/** Mod-drift check: compare the game's CURRENT mod set (live from the mods dir)
- * against the baseline this project's data was dumped from (#28, `meta.mod_list`),
- * by name AND version. Returns the categorized drift plus `needsRedump` — the
- * signal that the reference data no longer matches the game and a re-dump is due.
+/** Reference-data drift check: compare the game's CURRENT mod set (live from the
+ * mods dir) against the baseline this project's data was dumped from (#28,
+ * `meta.mod_list`), and compare the importer format recorded in project metadata
+ * against the app's current reader. Returns both causes plus `needsRedump` — the
+ * shared signal that the reference data needs to be rebuilt.
  * Cheap (two small file reads), so it's safe to poll on app start, on project
  * switch (a full reload re-runs it), on bridge reconnect, and periodically. */
 export const modDriftFn = createServerFn({ method: "GET" }).handler(async () => {
   const metaMap = q.metaAll();
+  const dataFormat = referenceDataFormatStatus(metaMap.data_format_version, q.stats().recipes > 0);
   let baseline: dump.ModEntry[] | null = null;
   if (metaMap.mod_list) {
     try {
@@ -1115,12 +1118,14 @@ export const modDriftFn = createServerFn({ method: "GET" }).handler(async () =>
   } catch {
     current = null; // factorio dir missing — can't compare, don't nag
   }
-  if (!baseline || !current)
-    return { haveBaseline: !!baseline, drift: null, needsRedump: false } as const;
+  const drift = baseline && current ? dump.diffMods(baseline, current) : null;
+  const modsChanged = baseline && current ? dump.redumpNeeded(baseline, current) : false;
   return {
-    haveBaseline: true,
-    drift: dump.diffMods(baseline, current),
-    needsRedump: dump.redumpNeeded(baseline, current),
+    haveBaseline: !!baseline,
+    drift,
+    modsChanged,
+    dataFormat,
+    needsRedump: modsChanged || dataFormat.stale,
   };
 });