feat(skill-memory): embeddings + intent-scoped recall (P2)

Upgrade recall from flat recency×hit to a multi-rung cascade: intent +
model-matched embeddings → cosine blend across intent_embedding +
delta_embedding (relevance/recency/hit weights tunable per skill via
ranking_* frontmatter); intent + no model match → FTS5 fallback over the
content-linked skill_memory_fts vtable; empty → flat fallback.

- migration: delta_embedding + recall_count columns + skill_memory_fts FTS5 vtable.
- embed-on-write in insertSkillMemoryNote; delta-only semantic dedup.
- programmatic, no-LLM reembed pre-step for the distill-skill-memory dreamer task.
- read-side recall_count (distinct from write-side hit_count).
- canonical vector serde + dedup/ranking/FTS query helpers.

Author: Tehan

packages/plugin/src/features/magic-context/dreamer/runner.ts

@@ -377,47 +377,32 @@ const STRUCTURE_TEMPLATE = `
 // ── Distill Skill Memory ───────────────────────────────────────────────────
 
 function buildDistillSkillMemoryPrompt(projectPath: string): string {
-    return `## Task: Distill Skill Memory
+    return `## Task: Distill Skill Memory (P2 — read-only health report)
 
 **Project:** ${projectPath}
 
-### Goal
-Maintain the skill_memory table: merge near-duplicate notes, prune stale low-hit notes,
-promote recurring gotchas to pinned, enforce per-skill note caps.
+### Important context
+- Embedding refresh for NULL/stale vectors already ran programmatically BEFORE this prompt — no need to re-embed.
+- Merge (action="distill" + merge), prune, and promote are P3 / NOT YET IMPLEMENTED. Do NOT call ctx_skill_note with action="distill".
 
-### Process
-1. Query skill_memory for skills with note_count > 20 (the distill threshold):
+### Your task: produce a short read-only summary of skill-memory corpus health
+1. Query aggregate counts and flag obvious issues:
    \`\`\`sql
-   SELECT skill_id, tier, project_identity, COUNT(*) as note_count
+   SELECT skill_id, tier, COUNT(*) as note_count,
+          SUM(CASE WHEN pinned = 1 THEN 1 ELSE 0 END) as pinned_count,
+          SUM(CASE WHEN intent_embedding IS NULL OR delta_embedding IS NULL THEN 1 ELSE 0 END) as missing_embedding_count
    FROM skill_memory
    WHERE project_identity = (SELECT project_identity FROM skill_memory LIMIT 1)
-   GROUP BY skill_id, tier, project_identity
-   HAVING note_count > 20
+   GROUP BY skill_id, tier
    ORDER BY note_count DESC
-   LIMIT 5;
+   LIMIT 20;
    \`\`\`
-2. For each qualifying skill:
-   a. List notes ordered by hit_count DESC, created_at DESC.
-   b. Merge near-duplicate notes (same kind, similar delta — use judgment).
-      Use ctx_skill_note with action="distill" and merge: [id, id].
-   c. Prune notes with hit_count=0 AND created_at < now-30d (stale, never recalled).
-      Use ctx_skill_note with action="distill" and prune: id.
-   d. Promote notes with hit_count >= 5 to pinned=1 if not already pinned.
-      Use ctx_skill_note with action="distill" and promote: id.
-   e. If note count > 100 after pruning, archive oldest low-hit unpinned notes.
-3. Log a quality alert if >30% of kind='gotcha' notes appear to be general observations
-   (not skill-specific). Use ctx_memory to record the alert.
-4. Process at most 5 skill groups per run (rotating by last_distilled_at).
-
-### Tools available
-- ctx_skill_note (with action="distill" — dreamer-only action for merge/prune/promote)
-- Read, bash (for verification queries)
+2. Note any skills with >100 notes, >30% gotcha-kind notes, or obvious near-duplicates (same skill + kind + very similar delta text).
+3. Report findings as a short summary — no tool calls beyond read-only SQL queries.
 
 ### Success criteria
-- No skill has >100 notes in the project tier.
-- Pinned notes reflect genuinely recurring gotchas (hit_count >= 5).
-- Stale zero-hit notes older than 30 days are pruned.
-- Quality alert logged if >30% of gotcha notes are general-quality.`;
+- A concise health summary is logged (via ctx_memory) for the project maintainer to review.
+- No tool calls to unimplemented actions.`;
 }
 
 // ── Dispatcher ─────────────────────────────────────────────────────────────

packages/plugin/src/features/magic-context/dreamer/task-prompts.ts

@@ -0,0 +1,11 @@
+import { describe, expect, test } from "bun:test";
+import { float32ArrayToBlob, toFloat32Array } from "./storage-memory-embeddings";
+
+describe("vector serde round-trip", () => {
+    test("Float32Array → blob → Float32Array preserves values", () => {
+        const vec = new Float32Array([0.1, -0.5, 0.99, 0.0]);
+        const blob = float32ArrayToBlob(vec);
+        const back = toFloat32Array(blob);
+        expect(Array.from(back)).toEqual(Array.from(vec));
+    });
+});

packages/plugin/src/features/magic-context/memory/storage-memory-embeddings.test.ts

@@ -37,7 +37,7 @@ function isEmbeddingRow(row: unknown): row is EmbeddingRow {
     );
 }
 
-function toFloat32Array(blob: Uint8Array | ArrayBuffer): Float32Array {
+export function toFloat32Array(blob: Uint8Array | ArrayBuffer): Float32Array {
     if (blob instanceof Uint8Array) {
         const buffer = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength);
         return new Float32Array(buffer);
@@ -46,6 +46,11 @@ function toFloat32Array(blob: Uint8Array | ArrayBuffer): Float32Array {
     return new Float32Array(blob.slice(0));
 }
 
+/** Serialize a Float32Array to a SQLite BLOB (Buffer). Canonical — reuse, do not duplicate. */
+export function float32ArrayToBlob(vector: Float32Array): Buffer {
+    return Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength);
+}
+
 function getSaveEmbeddingStatement(db: Database): PreparedStatement {
     let stmt = saveEmbeddingStatements.get(db);
     if (!stmt) {

packages/plugin/src/features/magic-context/memory/storage-memory-embeddings.ts

@@ -47,7 +47,12 @@ function assertForeignKeyIntegrity(db: Database, table?: string): void {
     }
 }
 
-const MIGRATIONS: Migration[] = [
+function columnExists(db: Database, table: string, column: string): boolean {
+    const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name?: string }>;
+    return rows.some((row) => row.name === column);
+}
+
+export const MIGRATIONS: Migration[] = [
     {
         version: 1,
         description: "Merge session_notes + smart_notes into unified notes table",
@@ -1918,6 +1923,58 @@ const MIGRATIONS: Migration[] = [
             `);
         },
     },
+
+    {
+        // Skill-memory P2: was v39/v43 across earlier rebases; renumbered to v51
+        // after upstream v0.27 took v42–v49 (skill-P1 is now v50).
+        version: 51,
+        description:
+            "Skill-memory P2: delta_embedding + recall_count columns + skill_memory_fts FTS5 vtable",
+        up: (db: Database) => {
+            // skill_memory is migration-only (created by v50); ALTER is safe here.
+            if (!columnExists(db, "skill_memory", "delta_embedding")) {
+                db.exec(`ALTER TABLE skill_memory ADD COLUMN delta_embedding BLOB;`);
+            }
+
+            // recall_count: read-side usage counter, bumped each time a note is surfaced
+            // in a recall block (distinct from hit_count, which is write-side re-record salience).
+            // Answers "which notes are recalled most". NOT_NULL+DEFAULT is valid in ALTER ADD COLUMN.
+            if (!columnExists(db, "skill_memory", "recall_count")) {
+                db.exec(
+                    `ALTER TABLE skill_memory ADD COLUMN recall_count INTEGER NOT NULL DEFAULT 0;`,
+                );
+            }
+
+            // FTS5 over (intent, delta), content-linked to skill_memory — mirrors memories_fts.
+            db.exec(`
+                CREATE VIRTUAL TABLE IF NOT EXISTS skill_memory_fts USING fts5(
+                  intent,
+                  delta,
+                  content='skill_memory',
+                  content_rowid='id',
+                  tokenize='porter unicode61'
+                );
+
+                CREATE TRIGGER IF NOT EXISTS skill_memory_ai AFTER INSERT ON skill_memory BEGIN
+                  INSERT INTO skill_memory_fts(rowid, intent, delta) VALUES (new.id, new.intent, new.delta);
+                END;
+
+                CREATE TRIGGER IF NOT EXISTS skill_memory_ad AFTER DELETE ON skill_memory BEGIN
+                  INSERT INTO skill_memory_fts(skill_memory_fts, rowid, intent, delta) VALUES ('delete', old.id, old.intent, old.delta);
+                END;
+
+                CREATE TRIGGER IF NOT EXISTS skill_memory_au AFTER UPDATE ON skill_memory BEGIN
+                  INSERT INTO skill_memory_fts(skill_memory_fts, rowid, intent, delta) VALUES ('delete', old.id, old.intent, old.delta);
+                  INSERT INTO skill_memory_fts(rowid, intent, delta) VALUES (new.id, new.intent, new.delta);
+                END;
+            `);
+
+            // Backfill the FTS index for any existing skill_memory rows. External-content FTS5 tables
+            // expose content rowids immediately, so a `NOT IN (SELECT rowid FROM …_fts)` guard is a no-op;
+            // the 'rebuild' command is the correct way to (re)populate an external-content index.
+            db.exec(`INSERT INTO skill_memory_fts(skill_memory_fts) VALUES('rebuild');`);
+        },
+    },
 ];
 
 /**

packages/plugin/src/features/magic-context/migrations.ts

@@ -44,4 +44,29 @@ describe("parseFrontmatterConfig", () => {
         const content = `---\nskill-memory: true\n---\n# Skill`;
         expect(parseFrontmatterConfig(content)).toBeNull();
     });
+
+    test("parses flat ranking_* keys as numbers", () => {
+        const md = `---
+skill-memory:
+  enabled: true
+  ranking_relevance: 0.7
+  ranking_recency: 0.2
+  ranking_hit: 0.1
+---
+body`;
+        const cfg = parseFrontmatterConfig(md);
+        expect(cfg?.ranking_relevance).toBe(0.7);
+        expect(cfg?.ranking_recency).toBe(0.2);
+        expect(cfg?.ranking_hit).toBe(0.1);
+    });
+
+    test("ranking_* default to undefined when omitted (recall applies defaults)", () => {
+        const md = `---
+skill-memory:
+  enabled: true
+---
+body`;
+        const cfg = parseFrontmatterConfig(md);
+        expect(cfg?.ranking_relevance).toBeUndefined();
+    });
 });

packages/plugin/src/features/magic-context/skill-memory/frontmatter.test.ts

@@ -11,6 +11,9 @@ export interface SkillMemoryConfig {
     max_tokens: number;
     max_pinned_tokens: number;
     dedup_threshold: number;
+    ranking_relevance?: number;
+    ranking_recency?: number;
+    ranking_hit?: number;
 }
 
 const FRONTMATTER_REGEX = /^---\r?\n([\s\S]*?)\r?\n---/m;
@@ -32,6 +35,9 @@ export function parseFrontmatterConfig(content: string): SkillMemoryConfig | nul
             max_tokens: toNumber(skillMemoryBlock.max_tokens, 1500),
             max_pinned_tokens: toNumber(skillMemoryBlock.max_pinned_tokens, 4000),
             dedup_threshold: toNumber(skillMemoryBlock.dedup_threshold, 0.92),
+            ranking_relevance: toOptionalNumber(skillMemoryBlock.ranking_relevance),
+            ranking_recency: toOptionalNumber(skillMemoryBlock.ranking_recency),
+            ranking_hit: toOptionalNumber(skillMemoryBlock.ranking_hit),
         };
     } catch {
         // Non-choke: malformed config = inert
@@ -48,6 +54,15 @@ function toNumber(value: unknown, defaultValue: number): number {
     return defaultValue;
 }
 
+function toOptionalNumber(value: unknown): number | undefined {
+    if (typeof value === "number" && Number.isFinite(value)) return value;
+    if (typeof value === "string") {
+        const parsed = Number(value);
+        if (Number.isFinite(parsed)) return parsed;
+    }
+    return undefined;
+}
+
 /**
  * Extract the `skill-memory:` sub-block from YAML frontmatter text.
  * Returns a flat key→value map of the block's immediate children.