mason: backfill historical shadow embeddings on Synapse identity rotation

The shadow lane only mirrored NEW primary writes, so a fingerprint rotation
gave the shadow modelId a clean slate: the entire historical corpus kept its
rows under the OLD identity while the new one had none. The 14-day GC then
deleted the old rows, silently zeroing the shadow measurement cohort right
before the phase-2 cutover gate (rank-overlap over real queries) needs it.

registerProjectShadowEmbedding now detects a freshly landed shadow identity
(a rotation, or a first/again registration over a corpus that already has
primary rows but no shadow rows under the current identity) and arms a
historical backfill. For each scope (memory/commit/chunk) it finds items the
primary lane embedded that lack a row under the new shadow modelId — the same
LEFT JOIN ... WHERE NULL shape as the primary backfill — and drains them
through the existing bounded shadow worker a chunk per tick, so registration
stays cheap and concurrency stays 1. A per-scope stall guard stops re-enqueue
when a batch makes no progress, keeping a failing provider from hot-looping.
The untrusted-config latch gates the enqueue, and nothing changes when shadow
embedding is disabled or the identity is unchanged.

Also adds scripts/backfill-embeddings.ts --shadow to drive the same enqueue
path to completion with progress output, for firing a backlog manually.

Author: ualtinok

packages/plugin/scripts/backfill-embeddings.ts

@@ -9,23 +9,98 @@
  * Run: bun scripts/backfill-embeddings.ts [--directory <cwd>] [--project <project_path>]
  *   --directory  Project directory used to resolve config and identity.
  *   --project    Only backfill memories for this project_path (must match --directory identity unless --force-project-path).
+ *   --shadow     Re-embed the HISTORICAL corpus under the current Synapse shadow
+ *                identity instead of backfilling primary memories. Use this after
+ *                a fingerprint rotation orphaned the shadow measurement cohort;
+ *                it drives the same bounded enqueue path the plugin runs, but to
+ *                completion with progress output.
  */
 import { Database } from "../src/shared/sqlite";
-import { loadPluginConfig } from "../src/config";
+import { loadPluginConfig, loadPluginConfigDetailed } from "../src/config";
 import {
     embedBatchForProject,
+    flushShadowEmbeddingBacklog,
     getProjectEmbeddingSnapshot,
+    getShadowBackfillRemaining,
     registerProjectEmbedding,
+    registerProjectShadowEmbedding,
 } from "../src/features/magic-context/memory/embedding";
 import { resolveProjectIdentity } from "../src/features/magic-context/memory/project-identity";
 import { saveEmbedding } from "../src/features/magic-context/memory/storage-memory-embeddings";
+import { isConfigLoadUntrusted } from "../src/plugin/embedding-bootstrap-helpers";
+import { resolveEmbeddingRouting } from "../src/plugin/embedding-routing";
 
 const DB_PATH = `${process.env.HOME}/.local/share/opencode/storage/plugin/magic-context/context.db`;
 function getArg(name: string): string | null {
     const index = process.argv.indexOf(name);
     return index >= 0 ? (process.argv[index + 1] ?? null) : null;
 }
 
+/**
+ * Drive the shadow historical backfill to completion with progress output.
+ * Registering the (possibly rotated) shadow identity arms the backfill inside
+ * the registry; this just resolves the live routing, registers both lanes the
+ * same way the boot path does, then flushes the bounded queue until the missing
+ * set is empty.
+ */
+async function runShadowBackfill(
+    db: Database,
+    directory: string,
+    projectIdentity: string,
+): Promise<void> {
+    const detailed = loadPluginConfigDetailed(directory);
+    if (isConfigLoadUntrusted(detailed)) {
+        console.error(
+            "Config load is untrusted for this project; refusing to run the shadow backfill off a config we don't trust.",
+        );
+        process.exitCode = 1;
+        return;
+    }
+    const routing = await resolveEmbeddingRouting({
+        config: detailed.config,
+        projectRoot: directory,
+        session: `shadow-backfill:${projectIdentity}`,
+    });
+    for (const warning of routing.warnings) console.warn(`[shadow] ${warning}`);
+    if (!routing.shadow) {
+        console.error(
+            "No shadow lane is armed for this project (shadow_embedding disabled or unavailable). Nothing to do.",
+        );
+        return;
+    }
+    registerProjectEmbedding(
+        db,
+        projectIdentity,
+        routing.primary,
+        {
+            memoryEnabled: detailed.config.memory.enabled,
+            gitCommitEnabled: detailed.config.memory.git_commit_indexing.enabled,
+        },
+        directory,
+    );
+    registerProjectShadowEmbedding(db, projectIdentity, routing.shadow, directory);
+
+    const before = getShadowBackfillRemaining(db, projectIdentity);
+    console.log(
+        `Shadow backfill for ${projectIdentity}: ${before.memory} memories, ${before.commit} commits, ${before.chunk} chunks missing under the current shadow identity.`,
+    );
+    if (before.memory === 0 && before.commit === 0 && before.chunk === 0) {
+        console.log("Nothing to do.");
+        return;
+    }
+    await flushShadowEmbeddingBacklog(projectIdentity, () => {
+        const remaining = getShadowBackfillRemaining(db, projectIdentity);
+        console.log(
+            `  remaining: ${remaining.memory} memories, ${remaining.commit} commits, ${remaining.chunk} chunks`,
+        );
+    });
+    const after = getShadowBackfillRemaining(db, projectIdentity);
+    console.log(
+        `Done. Embedded ${before.memory - after.memory} memories, ${before.commit - after.commit} commits, ${before.chunk - after.chunk} chunks. ` +
+            `Remaining: ${after.memory} memories, ${after.commit} commits, ${after.chunk} chunks.`,
+    );
+}
+
 async function main() {
     const directory = getArg("--directory") ?? process.cwd();
     const projectFilter = getArg("--project");
@@ -42,6 +117,13 @@ async function main() {
 
     const db = new Database(DB_PATH);
     db.exec("PRAGMA journal_mode=WAL");
+
+    if (process.argv.includes("--shadow")) {
+        await runShadowBackfill(db, directory, projectIdentity);
+        db.close();
+        return;
+    }
+
     const config = loadPluginConfig(directory);
     registerProjectEmbedding(
         db,

packages/plugin/src/features/magic-context/memory/embedding.ts

@@ -24,8 +24,10 @@ export {
     embedUnembeddedCompartmentChunksForProject,
     embedUnembeddedMemoriesForProject,
     enqueueShadowEmbeddingItems,
+    flushShadowEmbeddingBacklog,
     getPrimaryEmbeddingMeasurementCohort,
     getProjectEmbeddingSnapshot,
+    getShadowBackfillRemaining,
     getShadowEmbeddingMeasurementCohort,
     markProjectLoadUntrusted,
     registerProjectEmbedding,