feat(skill-memory): historian-extraction foundation — global '*' unification (P3a)

Foundation for the historian to auto-capture skill notes cross-project.

- surface the skill name in the historian chunk as a `TC: skill(<name>)`
  marker (the keystone — the tool input name was previously dropped).
- migration: origin_project + source_type columns; unify global-tier notes
  under project_identity='*' (collision-merge) so a global note is one row
  recallable from any repo.
- partitionKey helper routes global write/recall/reembed/stats through '*';
  recall reads global-tier from '*' (cross-project); reembed sweeps '*'.

Author: Tehan

packages/plugin/src/features/magic-context/migrations-v41.test.ts

@@ -0,0 +1,185 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "../../shared/sqlite";
+import { closeQuietly } from "../../shared/sqlite-helpers";
+import { LATEST_MIGRATION_VERSION, MIGRATIONS, runMigrations } from "./migrations";
+import { initializeDatabase, LATEST_SUPPORTED_VERSION } from "./storage-db";
+
+function migratedDb(): Database {
+    const db = new Database(":memory:");
+    initializeDatabase(db);
+    runMigrations(db);
+    return db;
+}
+
+function insertGlobal(
+    db: Database,
+    skillId: string,
+    projectIdentity: string,
+    hash: string,
+    opts: { hit?: number; recall?: number; lastUsed?: number | null; createdAt?: number } = {},
+): void {
+    db.prepare(
+        `INSERT INTO skill_memory (skill_id, resolved_path, tier, project_identity, intent, kind, delta, hit_count, recall_count, pinned, normalized_hash, created_at, last_used_at)
+         VALUES (?, '/p', 'global', ?, 'i', 'fix', 'd-' || ?, ?, ?, 0, ?, ?, ?)`,
+    ).run(
+        skillId,
+        projectIdentity,
+        hash,
+        opts.hit ?? 0,
+        opts.recall ?? 0,
+        hash,
+        opts.createdAt ?? Date.now(),
+        opts.lastUsed ?? null,
+    );
+}
+
+describe("migration v41 — origin_project + source_type + global '*' unification", () => {
+    test("LATEST_SUPPORTED_VERSION equals LATEST_MIGRATION_VERSION after v41", () => {
+        expect(LATEST_SUPPORTED_VERSION).toBe(41);
+        expect(LATEST_MIGRATION_VERSION).toBe(41);
+    });
+
+    test("fresh DB has origin_project + source_type columns", () => {
+        const db = migratedDb();
+        try {
+            const cols = (
+                db.prepare("PRAGMA table_info(skill_memory)").all() as Array<{ name: string }>
+            ).map((r) => r.name);
+            expect(cols).toContain("origin_project");
+            expect(cols).toContain("source_type");
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("singleton global note rewritten to '*' with origin_project preserved", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            for (const m of MIGRATIONS.filter((m) => m.version <= 40)) m.up(db);
+            insertGlobal(db, "council", "git:repoA", "h1");
+
+            const v41 = MIGRATIONS.find((m) => m.version === 41);
+            expect(v41).toBeDefined();
+            v41?.up(db);
+
+            const row = db
+                .prepare(
+                    "SELECT project_identity, origin_project FROM skill_memory WHERE normalized_hash='h1'",
+                )
+                .get() as { project_identity: string; origin_project: string };
+            expect(row.project_identity).toBe("*");
+            expect(row.origin_project).toBe("git:repoA");
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("collision-merge: same lesson in 2 repos → one '*' row, summed counters, MAX(last_used_at)", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            for (const m of MIGRATIONS.filter((m) => m.version <= 40)) m.up(db);
+            insertGlobal(db, "council", "git:repoA", "dup", {
+                hit: 2,
+                recall: 3,
+                lastUsed: 1000,
+                createdAt: 500,
+            });
+            insertGlobal(db, "council", "git:repoB", "dup", {
+                hit: 5,
+                recall: 1,
+                lastUsed: 9000,
+                createdAt: 800,
+            });
+
+            MIGRATIONS.find((m) => m.version === 41)?.up(db);
+
+            const rows = db
+                .prepare(
+                    "SELECT project_identity, hit_count, recall_count, last_used_at, created_at FROM skill_memory WHERE normalized_hash='dup'",
+                )
+                .all() as Array<{
+                project_identity: string;
+                hit_count: number;
+                recall_count: number;
+                last_used_at: number;
+                created_at: number;
+            }>;
+            expect(rows.length).toBe(1);
+            expect(rows[0].project_identity).toBe("*");
+            expect(rows[0].hit_count).toBe(7);
+            expect(rows[0].recall_count).toBe(4);
+            expect(rows[0].last_used_at).toBe(9000);
+            expect(rows[0].created_at).toBe(500);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("idempotent: re-running v41 up() does not double-process '*' rows", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            for (const m of MIGRATIONS.filter((m) => m.version <= 40)) m.up(db);
+            insertGlobal(db, "council", "git:repoA", "h1", { hit: 1 });
+
+            const v41 = MIGRATIONS.find((m) => m.version === 41);
+            v41?.up(db);
+            v41?.up(db);
+
+            const rows = db
+                .prepare("SELECT hit_count FROM skill_memory WHERE normalized_hash='h1'")
+                .all() as Array<{ hit_count: number }>;
+            expect(rows.length).toBe(1);
+            expect(rows[0].hit_count).toBe(1);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("FTS index consistent after collision-merge (no orphans)", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            for (const m of MIGRATIONS.filter((m) => m.version <= 40)) m.up(db);
+            insertGlobal(db, "council", "git:repoA", "dup");
+            insertGlobal(db, "council", "git:repoB", "dup");
+
+            MIGRATIONS.find((m) => m.version === 41)?.up(db);
+
+            const ftsCount = db.prepare("SELECT COUNT(*) AS n FROM skill_memory_fts").get() as {
+                n: number;
+            };
+            const rowCount = db.prepare("SELECT COUNT(*) AS n FROM skill_memory").get() as {
+                n: number;
+            };
+            // Prove the merge actually happened (2 dup rows → 1) so the parity
+            // assertion below isn't trivially true on a no-op merge.
+            expect(rowCount.n).toBe(1);
+            expect(ftsCount.n).toBe(rowCount.n);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("project-tier rows untouched", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            for (const m of MIGRATIONS.filter((m) => m.version <= 40)) m.up(db);
+            db.prepare(
+                `INSERT INTO skill_memory (skill_id, resolved_path, tier, project_identity, intent, kind, delta, normalized_hash, created_at) VALUES ('s', '/p', 'project', 'git:repoA', 'i', 'fix', 'd', 'ph', 1)`,
+            ).run();
+
+            MIGRATIONS.find((m) => m.version === 41)?.up(db);
+
+            const row = db
+                .prepare("SELECT project_identity FROM skill_memory WHERE normalized_hash='ph'")
+                .get() as { project_identity: string };
+            expect(row.project_identity).toBe("git:repoA");
+        } finally {
+            closeQuietly(db);
+        }
+    });
+});

packages/plugin/src/features/magic-context/migrations.ts

@@ -1975,6 +1975,78 @@ export const MIGRATIONS: Migration[] = [
             db.exec(`INSERT INTO skill_memory_fts(skill_memory_fts) VALUES('rebuild');`);
         },
     },
+    {
+        // Skill-memory historian extraction: was v41/v44 across earlier rebases;
+        // renumbered to v52 after upstream v0.27 took v42–v49 (skill is now v50/51).
+        version: 52,
+        description:
+            "Skill-memory historian extraction: origin_project + source_type columns; unify global-tier notes under project_identity='*' (collision-merge)",
+        up: (db: Database) => {
+            db.transaction(() => {
+                if (!columnExists(db, "skill_memory", "origin_project")) {
+                    db.exec(`ALTER TABLE skill_memory ADD COLUMN origin_project TEXT;`);
+                }
+                if (!columnExists(db, "skill_memory", "source_type")) {
+                    db.exec(`ALTER TABLE skill_memory ADD COLUMN source_type TEXT;`);
+                }
+
+                // resolved_path stays TEXT NOT NULL; historian writes the '' sentinel
+                // (handled in storage layer, not here).
+                const groups = db
+                    .prepare(
+                        `SELECT skill_id, normalized_hash, COUNT(*) AS n, MIN(created_at) AS min_created,
+                                SUM(hit_count) AS sum_hit, SUM(recall_count) AS sum_recall, MAX(last_used_at) AS max_used
+                         FROM skill_memory
+                         WHERE tier='global' AND project_identity != '*'
+                         GROUP BY skill_id, normalized_hash HAVING COUNT(*) > 1`,
+                    )
+                    .all() as Array<{
+                    skill_id: string;
+                    normalized_hash: string;
+                    n: number;
+                    min_created: number;
+                    sum_hit: number;
+                    sum_recall: number;
+                    max_used: number | null;
+                }>;
+                for (const g of groups) {
+                    const survivor = db
+                        .prepare(
+                            `SELECT id, project_identity FROM skill_memory
+                             WHERE skill_id=? AND normalized_hash=? AND tier='global' AND project_identity != '*'
+                             ORDER BY created_at ASC, id ASC LIMIT 1`,
+                        )
+                        .get(g.skill_id, g.normalized_hash) as {
+                        id: number;
+                        project_identity: string;
+                    };
+                    db.prepare(
+                        `DELETE FROM skill_memory WHERE skill_id=? AND normalized_hash=? AND tier='global' AND project_identity != '*' AND id != ?`,
+                    ).run(g.skill_id, g.normalized_hash, survivor.id);
+                    db.prepare(
+                        `UPDATE skill_memory SET hit_count=?, recall_count=?, last_used_at=?, origin_project=?, project_identity='*' WHERE id=?`,
+                    ).run(g.sum_hit, g.sum_recall, g.max_used, survivor.project_identity, survivor.id);
+                }
+
+                // Defensive (S4): drop any pre-'*' row whose (skill_id, normalized_hash)
+                // already has a '*' sibling. Dead code in normal flow — v41 is the only
+                // writer of '*' rows and runs atomically, so a pre-'*' row can't coexist
+                // with a '*' sibling after a clean run. It only fires if a prior v41 run
+                // was interrupted after creating some '*' rows but before finishing; in
+                // that case the '*' row is canonical and the leftover pre-'*' row is
+                // dropped rather than colliding on the singleton UPDATE below.
+                db.prepare(
+                    `DELETE FROM skill_memory AS s
+                     WHERE s.tier='global' AND s.project_identity != '*'
+                       AND EXISTS (SELECT 1 FROM skill_memory g WHERE g.tier='global' AND g.project_identity='*' AND g.skill_id=s.skill_id AND g.normalized_hash=s.normalized_hash)`,
+                ).run();
+
+                db.prepare(
+                    `UPDATE skill_memory SET origin_project = project_identity, project_identity = '*' WHERE tier='global' AND project_identity != '*'`,
+                ).run();
+            })();
+        },
+    },
 ];
 
 /**

packages/plugin/src/features/magic-context/skill-memory/promote.test.ts

@@ -0,0 +1,61 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "../../../shared/sqlite";
+import { closeQuietly } from "../../../shared/sqlite-helpers";
+import { runMigrations } from "../migrations";
+import { initializeDatabase } from "../storage-db";
+import { promoteSkillObservations } from "./promote";
+
+function makeDb(): Database {
+    const db = new Database(":memory:");
+    initializeDatabase(db);
+    runMigrations(db);
+    return db;
+}
+
+describe("promoteSkillObservations", () => {
+    test("direct-writes a global '*' note with historian provenance", () => {
+        const db = makeDb();
+        try {
+            const n = promoteSkillObservations(db, "git:repoA", [
+                { skillId: "council", kind: "gotcha", lesson: "aggregator needs a fast model" },
+            ]);
+            expect(n).toBe(1);
+            const row = db.prepare("SELECT tier, project_identity, origin_project, source_type, resolved_path, kind FROM skill_memory").get() as Record<
+                string,
+                string
+            >;
+            expect(row.tier).toBe("global");
+            expect(row.project_identity).toBe("*");
+            expect(row.origin_project).toBe("git:repoA");
+            expect(row.source_type).toBe("historian");
+            expect(row.resolved_path).toBe("");
+            expect(row.kind).toBe("gotcha");
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("exact-hash duplicate bumps hit_count instead of inserting", () => {
+        const db = makeDb();
+        try {
+            promoteSkillObservations(db, "git:repoA", [{ skillId: "council", kind: "fix", lesson: "same lesson" }]);
+            const n = promoteSkillObservations(db, "git:repoB", [{ skillId: "council", kind: "fix", lesson: "same lesson" }]);
+            expect(n).toBe(0);
+            const rows = db.prepare("SELECT hit_count FROM skill_memory").all() as Array<{ hit_count: number }>;
+            expect(rows.length).toBe(1);
+            expect(rows[0].hit_count).toBe(1);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("rejects kind='general'", () => {
+        const db = makeDb();
+        try {
+            const n = promoteSkillObservations(db, "git:repoA", [{ skillId: "council", kind: "general" as never, lesson: "x" }]);
+            expect(n).toBe(0);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+});

packages/plugin/src/features/magic-context/skill-memory/promote.ts

@@ -0,0 +1,60 @@
+import type { Database } from "../../../shared/sqlite";
+import { computeNormalizedHash } from "../memory/normalize-hash";
+import { bumpHitCount, findExistingNote, insertSkillMemoryNote, partitionKey } from "./storage";
+
+const VALID_KINDS = new Set(["gotcha", "discovery", "fix", "workflow"]);
+
+export interface SkillObservation {
+    skillId: string;
+    kind: "gotcha" | "discovery" | "fix" | "workflow";
+    lesson: string;
+}
+
+/**
+ * Direct-write historian-extracted skill observations as GLOBAL-tier notes under
+ * the '*' partition (source_type='historian', resolved_path='' sentinel). Hash-dedup:
+ * an exact-hash match bumps hit_count instead of inserting. Returns the number of
+ * NEW notes written (dups excluded). Best-effort per item: never throws.
+ */
+export function promoteSkillObservations(
+    db: Database,
+    originProject: string,
+    observations: SkillObservation[],
+): number {
+    let written = 0;
+    const tier = "global" as const;
+    const part = partitionKey(tier, originProject);
+
+    for (const obs of observations) {
+        if (!obs.skillId || !obs.lesson || !VALID_KINDS.has(obs.kind)) continue;
+
+        try {
+            const normalizedHash = computeNormalizedHash(obs.lesson);
+            const existing = findExistingNote(db, obs.skillId, tier, part, normalizedHash);
+            if (existing) {
+                bumpHitCount(db, obs.skillId, tier, part, normalizedHash);
+                continue;
+            }
+
+            const id = insertSkillMemoryNote(db, {
+                skillId: obs.skillId,
+                resolvedPath: "",
+                tier,
+                skillSource: null,
+                projectIdentity: part,
+                originProject,
+                sourceType: "historian",
+                intent: obs.lesson,
+                kind: obs.kind,
+                delta: obs.lesson,
+                normalizedHash,
+                createdAt: Date.now(),
+            });
+            if (id !== null) written++;
+        } catch {
+            // Best-effort: one bad observation must not block the publish.
+        }
+    }
+
+    return written;
+}