feat(skill-memory): historian auto-extraction pipeline (P3b)

Close the loop so the historian writes skill notes during compaction
without an agent volunteering ctx_skill_note.

- historian prompt emits a <skill_observations> block; parser extracts it;
  threaded through the validated historian result.
- both runners (OpenCode + Pi) promote skill observations post-commit via
  the shared promoteSkillObservations helper, gated by
  promotionActive && !discardedLast, writing global '*' notes with
  source_type='historian'.
- self-heal net: initializeDatabase re-creates skill_memory + ensureColumn
  so an upgraded DB recovers even if a migration row is lost.

Author: Tehan

packages/docs/src/content/docs/reference/configuration.md

@@ -141,7 +141,7 @@ Off-hours maintenance (Dreamer) and on-demand prompt augmentation (Sidekick).
 | `dreamer.fallback_models` | string \\| string[] | — | Fallback model IDs if primary is unavailable |
 | `dreamer.schedule` | string | `"02:00-06:00"` | Scheduled window for overnight dreaming (e.g. '02:00-06:00') |
 | `dreamer.max_runtime_minutes` | number (10–) | `120` | Maximum runtime per dream session in minutes |
-| `dreamer.tasks` | `"consolidate"` \\| `"verify"` \\| `"archive-stale"` \\| `"improve"` \\| `"maintain-docs"`[] | `["consolidate","verify","archive-stale","improve"]` | Tasks to run during dreaming, in order |
+| `dreamer.tasks` | `"consolidate"` \\| `"verify"` \\| `"archive-stale"` \\| `"improve"` \\| `"maintain-docs"` \\| `"distill-skill-memory"`[] | `["consolidate","verify","archive-stale","improve"]` | Tasks to run during dreaming, in order |
 | `dreamer.task_timeout_minutes` | number (5–) | `20` | Minutes allocated per task before moving to next |
 | `dreamer.inject_docs` | boolean | `true` | Inject ARCHITECTURE.md and STRUCTURE.md into system prompt |
 | `dreamer.user_memories` | object | — | User memory pipeline: historian extracts behavior observations from each compartment run; dreamer reviews recurring patterns and promotes them to stable user memories injected into all sessions as <user-profile>. Requires dreamer to not be disabled for promotion to actually happen. Graduated from experimental in v0.14. Default: enabled. |

packages/pi-plugin/src/pi-historian-runner.test.ts

@@ -393,6 +393,25 @@ describe("runPiHistorian", () => {
 			closeQuietly(db);
 		}
 	});
+	it("promotes skillObservations as global '*' notes", async () => {
+		const xml = `${successXml()}\n<skill_observations>\n* council | gotcha | fast aggregator\n</skill_observations>`;
+		const { db } = await runHistorianWith({
+			outputs: [xml],
+			memoryEnabled: true,
+			autoPromote: true,
+		});
+		try {
+			const row = db
+				.prepare(
+					"SELECT project_identity, source_type FROM skill_memory WHERE skill_id='council'",
+				)
+				.get() as { project_identity: string; source_type: string } | undefined;
+			expect(row?.project_identity).toBe("*");
+			expect(row?.source_type).toBe("historian");
+		} finally {
+			closeQuietly(db);
+		}
+	});
 	it("runs the Pi subagent, parses output, and publishes compartments and facts", async () => {
 		const { db, runner } = await runHistorianWith({ outputs: [successXml()] });
 		try {

packages/pi-plugin/src/pi-historian-runner.ts

@@ -51,6 +51,7 @@ import {
 } from "@magic-context/core/features/magic-context/memory";
 import { resolveProjectIdentity } from "@magic-context/core/features/magic-context/memory/project-identity";
 import { getMemoriesByProject } from "@magic-context/core/features/magic-context/memory/storage-memory";
+import { promoteSkillObservations } from "@magic-context/core/features/magic-context/skill-memory/promote";
 import {
 	clearEmergencyDrainLatch,
 	clearEmergencyRecovery,
@@ -1031,6 +1032,24 @@ export async function runPiHistorian(deps: PiHistorianDeps): Promise<void> {
 				}
 			}
 
+			if (
+				promotionActive &&
+				!discardedLast &&
+				validatedPass.skillObservations &&
+				validatedPass.skillObservations.length > 0
+			) {
+				try {
+					const written = promoteSkillObservations(
+						db,
+						projectPath,
+						validatedPass.skillObservations,
+					);
+					sessionLog(sessionId, `promoted ${written} skill observation(s)`);
+				} catch (error) {
+					sessionLog(sessionId, "failed to promote skill observations:", error);
+				}
+			}
+
 			// Raw chunk embeddings: the ctx_search semantic substrate over session
 			// history. Fire-and-forget, best-effort, memory-gated.
 			if (embeddingActive) {
@@ -1185,6 +1204,13 @@ type ValidationOutcome =
 					: never
 				: never;
 			userObservations?: string[];
+			skillObservations?: ReturnType<
+				typeof validateHistorianOutput
+			> extends infer T
+				? T extends { ok: true; skillObservations?: infer S }
+					? S
+					: never
+				: never;
 			events?: ReturnType<typeof validateHistorianOutput> extends infer T
 				? T extends { ok: true; events?: infer E }
 					? E
@@ -1226,6 +1252,7 @@ async function validateHistorianResult(
 			compartments: validation.compartments,
 			facts: validation.facts,
 			userObservations: validation.userObservations,
+			skillObservations: validation.skillObservations,
 			events: validation.events,
 		};
 	}

packages/plugin/src/features/magic-context/migrations-v42.test.ts

@@ -0,0 +1,126 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "../../shared/sqlite";
+import { closeQuietly } from "../../shared/sqlite-helpers";
+import { LATEST_MIGRATION_VERSION, runMigrations } from "./migrations";
+import { initializeDatabase, LATEST_SUPPORTED_VERSION } from "./storage-db"; // ESM import (not require) — matches codebase pattern
+
+function columnNames(db: Database, table: string): string[] {
+    return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(
+        (c) => c.name,
+    );
+}
+
+function tableExists(db: Database, name: string): boolean {
+    return Boolean(
+        db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = ?").get(name),
+    );
+}
+
+describe("migration v42 — skill_memory table", () => {
+    test("creates skill_memory table with correct columns on fresh DB, idempotently", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            runMigrations(db);
+            runMigrations(db); // idempotency check
+
+            expect(tableExists(db, "skill_memory")).toBe(true);
+
+            const cols = columnNames(db, "skill_memory");
+            expect(cols).toContain("id");
+            expect(cols).toContain("skill_id");
+            expect(cols).toContain("resolved_path");
+            expect(cols).toContain("tier");
+            expect(cols).toContain("skill_source");
+            expect(cols).toContain("project_identity");
+            expect(cols).toContain("intent");
+            expect(cols).toContain("intent_embedding");
+            expect(cols).toContain("embedding_model_version");
+            expect(cols).toContain("kind");
+            expect(cols).toContain("delta");
+            expect(cols).toContain("tags");
+            expect(cols).toContain("hit_count");
+            expect(cols).toContain("pinned");
+            expect(cols).toContain("normalized_hash");
+            expect(cols).toContain("created_at");
+            expect(cols).toContain("last_used_at");
+
+            expect(
+                db
+                    .prepare("SELECT version FROM schema_migrations ORDER BY version DESC LIMIT 1")
+                    .get(),
+            ).toEqual({ version: LATEST_MIGRATION_VERSION });
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("skill_memory CHECK constraints reject invalid tier and kind values", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            runMigrations(db);
+
+            const insert = db.prepare(`
+                INSERT INTO skill_memory
+                  (skill_id, resolved_path, tier, project_identity, intent, kind, delta, normalized_hash, hit_count, pinned, created_at)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, 0, ?)
+            `);
+
+            // Valid row
+            expect(() =>
+                insert.run(
+                    "test-skill",
+                    "/path/SKILL.md",
+                    "project",
+                    "git:abc123",
+                    "test intent",
+                    "gotcha",
+                    "test delta",
+                    "hash1",
+                    Date.now(),
+                ),
+            ).not.toThrow();
+
+            // Invalid tier
+            expect(() =>
+                insert.run(
+                    "test-skill",
+                    "/path/SKILL.md",
+                    "invalid-tier",
+                    "git:abc123",
+                    "test intent",
+                    "gotcha",
+                    "test delta",
+                    "hash2",
+                    Date.now(),
+                ),
+            ).toThrow();
+
+            // Invalid kind
+            expect(() =>
+                insert.run(
+                    "test-skill",
+                    "/path/SKILL.md",
+                    "project",
+                    "git:abc123",
+                    "test intent",
+                    "general",
+                    "test delta",
+                    "hash3",
+                    Date.now(),
+                ),
+            ).toThrow();
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("LATEST_SUPPORTED_VERSION equals LATEST_MIGRATION_VERSION after v42", () => {
+        // This test will fail until storage-db.ts is bumped to 39.
+        // Belt-and-braces: mirrors schema-version-fence.test.ts but is co-located with the migration.
+        // If this feels redundant, keep it with this comment — co-location aids discoverability.
+        // NOTE: use ESM import at the top of the file (not require()) to match codebase pattern.
+        expect(LATEST_SUPPORTED_VERSION).toBe(LATEST_MIGRATION_VERSION);
+    });
+});

packages/plugin/src/features/magic-context/migrations-v43.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, test } from "bun:test";
+import { Database } from "../../shared/sqlite";
+import { closeQuietly } from "../../shared/sqlite-helpers";
+import { LATEST_MIGRATION_VERSION, MIGRATIONS, runMigrations } from "./migrations";
+import { initializeDatabase, LATEST_SUPPORTED_VERSION } from "./storage-db";
+
+function columnNames(db: Database, table: string): string[] {
+    return (db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name: string }>).map(
+        (c) => c.name,
+    );
+}
+function tableExists(db: Database, name: string): boolean {
+    return Boolean(
+        db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name = ?").get(name),
+    );
+}
+
+describe("migration v43 — skill_memory embeddings + FTS", () => {
+    test("fresh DB: delta_embedding column and skill_memory_fts exist, no throw", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            runMigrations(db);
+            runMigrations(db); // idempotency
+
+            expect(columnNames(db, "skill_memory")).toContain("delta_embedding");
+            expect(tableExists(db, "skill_memory_fts")).toBe(true);
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("FTS triggers keep skill_memory_fts in sync with skill_memory", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            runMigrations(db);
+            db.prepare(
+                `INSERT INTO skill_memory
+                   (skill_id, resolved_path, tier, project_identity, intent, kind, delta, normalized_hash, hit_count, pinned, created_at)
+                 VALUES (?,?,?,?,?,?,?,?,0,0,?)`,
+            ).run(
+                "s1",
+                "/p/SKILL.md",
+                "global",
+                "git:abc",
+                "fix a flaky auth test",
+                "fix",
+                "mock Date.now in auth tests",
+                "h1",
+                Date.now(),
+            );
+
+            const hit = db
+                .prepare(
+                    `SELECT m.id FROM skill_memory_fts f JOIN skill_memory m ON m.id = f.rowid
+                     WHERE skill_memory_fts MATCH ?`,
+                )
+                .get('"auth"');
+            expect(hit).toBeTruthy();
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    // requires `import { MIGRATIONS } from "./migrations";` (added above)
+    test("v43 migration backfills FTS for rows that pre-existed v40", () => {
+        const db = new Database(":memory:");
+        try {
+            initializeDatabase(db);
+            // Build a PRE-v40 schema: apply every migration BELOW v40 directly via up() (runMigrations has no
+            // target-version param). skill_memory is created at v39; the FTS table + delta_embedding do NOT exist yet.
+            for (const m of MIGRATIONS.filter((x) => x.version < 43).sort(
+                (a, b) => a.version - b.version,
+            )) {
+                m.up(db);
+            }
+            // Insert a row under the pre-v43 schema — no FTS table yet, so no AFTER-INSERT trigger indexes it.
+            db.prepare(
+                `INSERT INTO skill_memory
+                   (skill_id, resolved_path, tier, project_identity, intent, kind, delta, normalized_hash, hit_count, pinned, created_at)
+                 VALUES (?,?,?,?,?,?,?,?,0,0,?)`,
+            ).run(
+                "s2",
+                "/p/SKILL.md",
+                "global",
+                "git:abc",
+                "handle oauth refresh",
+                "fix",
+                "rotate the token early",
+                "h2",
+                Date.now(),
+            );
+            // Apply ONLY v40's up() — its body must ALTER + create the FTS table + BACKFILL the pre-existing row.
+            const v43 = MIGRATIONS.find((m) => m.version === 43);
+            if (!v43) throw new Error("v40 migration not found");
+            v43.up(db);
+            const hit = db
+                .prepare(
+                    `SELECT m.id FROM skill_memory_fts f JOIN skill_memory m ON m.id = f.rowid WHERE skill_memory_fts MATCH ?`,
+                )
+                .get('"oauth"');
+            expect(hit).toBeTruthy();
+        } finally {
+            closeQuietly(db);
+        }
+    });
+
+    test("LATEST_SUPPORTED_VERSION equals LATEST_MIGRATION_VERSION after v43", () => {
+        expect(LATEST_SUPPORTED_VERSION).toBe(LATEST_MIGRATION_VERSION);
+        expect(LATEST_SUPPORTED_VERSION).toBe(44);
+    });
+});