Merge pull request #3 from ZengLiangYi/codex/skill-mcp-memory-loop

feat: add task memory recall and writeback loop

Author: ZengLiangYi

CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [0.4.6] - 2026-04-16
+
+### Memory Loop
+
+- **Task memory recall + writeback APIs** — Added `/api/memory/recall` and `/api/memory/writeback` with shared request/response contracts for project-first recall, global supplement recall, and agent/manual memory writeback.
+- **MCP memory tools** — Added `recall_for_task` and `write_task_memory` to the MCP server so external agents can retrieve and persist task memories through the same server-side contract.
+- **Conservative writeback semantics** — Automatic writeback now supports idempotent receipts, pending/completed indexing status, conservative merge decisions, and supplemental relations for strongly related memories.
+
+### Memory Metadata
+
+- **Project-scoped memory model** — Notes now store `project_key`, `scope`, `source_type`, `source_agent`, `task_kind`, `error_signatures`, `files_touched`, and `outcome_type`, with DB bootstrap/backfill for legacy imported notes.
+- **Synthetic memory origins** — Added synthetic origin conversations for writeback-created memories while hiding them from the default conversations list.
+- **Project identity stability** — Added canonical project-key derivation plus alias support so moved repos and upgraded project identifiers can still recall the same project memories.
+
+### Quality
+
+- **Coverage for memory services and routes** — Added regression tests across schemas, project-key derivation, recall, writeback, backfill, origin creation, decision logic, and HTTP route behavior.
+- **Real HTTP + MCP smoke coverage** — Verified the full writeback/recall loop through a real Fastify process and MCP stdio server with a mock embedding backend.
+
 ## [0.4.0] - 2026-04-10
 
 ### New Data Sources

server/src/cli/client.ts

@@ -2,6 +2,12 @@ import { spawn } from 'node:child_process';
 import { resolve } from 'node:path';
 import { homedir } from 'node:os';
 import { writeFileSync, readFileSync, existsSync, unlinkSync, openSync, mkdirSync } from 'node:fs';
+import type {
+  RecallForTaskRequest,
+  RecallForTaskResponse,
+  WriteTaskMemoryRequest,
+  WriteTaskMemoryResponse,
+} from '@chatcrystal/shared';
 
 function getDataDir(): string {
   if (import.meta.dirname.includes('node_modules')) {
@@ -303,6 +309,18 @@ export class CrystalClient {
     }>>('GET', `/api/notes/${id}/relations`);
   }
 
+  async recallForTask(body: RecallForTaskRequest) {
+    return this.request<RecallForTaskResponse>('POST', '/api/memory/recall', body);
+  }
+
+  async writeTaskMemory(body: WriteTaskMemoryRequest) {
+    return this.request<WriteTaskMemoryResponse>(
+      'POST',
+      '/api/memory/writeback',
+      body,
+    );
+  }
+
   async listTags() {
     return this.request<Array<{ id: number; name: string; count: number }>>('GET', '/api/tags');
   }

server/src/cli/mcp/server.ts

@@ -2,6 +2,10 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { z } from 'zod';
 import { CrystalClient } from '../client.js';
+import {
+  RecallForTaskRequestShape,
+  WriteTaskMemoryRequestShape,
+} from '../../services/memory/schemas.js';
 
 export async function startMcpServer(baseUrl: string) {
   const client = new CrystalClient(baseUrl);
@@ -87,6 +91,36 @@ export async function startMcpServer(baseUrl: string) {
     },
   );
 
+  server.tool(
+    'recall_for_task',
+    'Recall project-first and global-supplement memories for a task.',
+    RecallForTaskRequestShape,
+    async (input) => {
+      const data = await client.recallForTask(input);
+      return {
+        content: [{
+          type: 'text' as const,
+          text: JSON.stringify(data, null, 2),
+        }],
+      };
+    },
+  );
+
+  server.tool(
+    'write_task_memory',
+    'Persist a task memory with idempotent auto-writeback semantics.',
+    WriteTaskMemoryRequestShape,
+    async (input) => {
+      const data = await client.writeTaskMemory(input);
+      return {
+        content: [{
+          type: 'text' as const,
+          text: JSON.stringify(data, null, 2),
+        }],
+      };
+    },
+  );
+
   // Start stdio transport
   const transport = new StdioServerTransport();
   await server.connect(transport);

server/src/db/index.ts

@@ -2,12 +2,21 @@ import initSqlJs, { type Database } from 'sql.js';
 import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs';
 import { resolve, dirname, join } from 'node:path';
 import { appConfig } from '../config.js';
+import { backfillImportedNoteMetadata } from '../services/memory/backfill.js';
 import { SCHEMA_SQL } from './schema.js';
 
 let db: Database | null = null;
 
 const DB_PATH = resolve(appConfig.dataDir, 'chatcrystal.db');
 
+function ensureColumn(db: Database, table: string, column: string, sql: string) {
+  const info = db.exec(`PRAGMA table_info(${table})`);
+  const columns = info[0]?.values.map((row) => String(row[1])) ?? [];
+  if (!columns.includes(column)) {
+    db.run(sql);
+  }
+}
+
 export async function initDatabase(): Promise<Database> {
   if (db) return db;
 
@@ -38,16 +47,26 @@ export async function initDatabase(): Promise<Database> {
   // Run schema migration
   db.exec(SCHEMA_SQL);
 
-  // Migrate: add embedding_status column if missing (for existing DBs)
-  const colCheck = db.exec("PRAGMA table_info(notes)");
-  const columns = colCheck[0]?.values.map((row) => row[1] as string) ?? [];
-  if (!columns.includes('embedding_status')) {
-    db.run("ALTER TABLE notes ADD COLUMN embedding_status TEXT DEFAULT 'pending'");
-    // Mark notes that already have embeddings as done
-    db.run(`UPDATE notes SET embedding_status = 'done'
-            WHERE id IN (SELECT DISTINCT note_id FROM embeddings)`);
-    console.log('[DB] Migrated: added embedding_status column to notes');
-  }
+  ensureColumn(db, 'notes', 'embedding_status', "ALTER TABLE notes ADD COLUMN embedding_status TEXT DEFAULT 'pending'");
+  ensureColumn(db, 'notes', 'project_key', 'ALTER TABLE notes ADD COLUMN project_key TEXT');
+  ensureColumn(db, 'notes', 'scope', "ALTER TABLE notes ADD COLUMN scope TEXT DEFAULT 'project'");
+  ensureColumn(db, 'notes', 'source_type', "ALTER TABLE notes ADD COLUMN source_type TEXT DEFAULT 'imported-conversation'");
+  ensureColumn(db, 'notes', 'source_agent', "ALTER TABLE notes ADD COLUMN source_agent TEXT DEFAULT 'unknown'");
+  ensureColumn(db, 'notes', 'task_kind', 'ALTER TABLE notes ADD COLUMN task_kind TEXT');
+  ensureColumn(db, 'notes', 'error_signatures', 'ALTER TABLE notes ADD COLUMN error_signatures TEXT');
+  ensureColumn(db, 'notes', 'files_touched', 'ALTER TABLE notes ADD COLUMN files_touched TEXT');
+  ensureColumn(db, 'notes', 'outcome_type', 'ALTER TABLE notes ADD COLUMN outcome_type TEXT');
+  ensureColumn(
+    db,
+    'writeback_receipts',
+    'index_status',
+    "ALTER TABLE writeback_receipts ADD COLUMN index_status TEXT DEFAULT 'pending'",
+  );
+
+  // Mark notes that already have embeddings as done
+  db.run(`UPDATE notes SET embedding_status = 'done'
+          WHERE id IN (SELECT DISTINCT note_id FROM embeddings)`);
+  backfillImportedNoteMetadata(db);
 
   // Persist to disk
   saveDatabase();

server/src/db/schema.ts

@@ -48,11 +48,40 @@ CREATE TABLE IF NOT EXISTS notes (
   raw_llm_response TEXT,
   is_edited INTEGER DEFAULT 0,
   embedding_status TEXT DEFAULT 'pending',
+  project_key TEXT,
+  scope TEXT DEFAULT 'project',
+  source_type TEXT DEFAULT 'imported-conversation',
+  source_agent TEXT DEFAULT 'unknown',
+  task_kind TEXT,
+  error_signatures TEXT,
+  files_touched TEXT,
+  outcome_type TEXT,
   created_at TEXT DEFAULT (datetime('now')),
   updated_at TEXT DEFAULT (datetime('now')),
   FOREIGN KEY (conversation_id) REFERENCES conversations(id) ON DELETE CASCADE
 );
 
+CREATE TABLE IF NOT EXISTS writeback_receipts (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  source_agent TEXT NOT NULL,
+  source_run_key TEXT NOT NULL,
+  decision TEXT NOT NULL,
+  note_id INTEGER,
+  merged_into_note_id INTEGER,
+  reason TEXT NOT NULL,
+  index_status TEXT NOT NULL DEFAULT 'pending',
+  created_at TEXT DEFAULT (datetime('now')),
+  UNIQUE(source_agent, source_run_key),
+  FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE SET NULL,
+  FOREIGN KEY (merged_into_note_id) REFERENCES notes(id) ON DELETE SET NULL
+);
+
+CREATE TABLE IF NOT EXISTS project_key_aliases (
+  alias_key TEXT PRIMARY KEY,
+  canonical_key TEXT NOT NULL,
+  created_at TEXT DEFAULT (datetime('now'))
+);
+
 CREATE TABLE IF NOT EXISTS tags (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   name TEXT UNIQUE NOT NULL
@@ -90,8 +119,10 @@ CREATE INDEX IF NOT EXISTS idx_conversations_source ON conversations(source);
 CREATE INDEX IF NOT EXISTS idx_conversations_status ON conversations(status);
 CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id);
 CREATE INDEX IF NOT EXISTS idx_messages_type ON messages(type);
+CREATE INDEX IF NOT EXISTS idx_notes_project_key ON notes(project_key);
 CREATE INDEX IF NOT EXISTS idx_note_tags_tag ON note_tags(tag_id);
 CREATE INDEX IF NOT EXISTS idx_embeddings_note ON embeddings(note_id);
+CREATE INDEX IF NOT EXISTS idx_project_key_aliases_canonical ON project_key_aliases(canonical_key);
 
 CREATE TABLE IF NOT EXISTS note_relations (
   id INTEGER PRIMARY KEY AUTOINCREMENT,

server/src/index.ts

@@ -16,6 +16,7 @@ import { conversationRoutes } from './routes/conversations.js';
 import { noteRoutes } from './routes/notes.js';
 import { configRoutes } from './routes/config.js';
 import { relationRoutes } from './routes/relations.js';
+import { memoryRoutes } from './routes/memory.js';
 
 // Initialize parser adapters (registers built-in adapters)
 import './parser/index.js';
@@ -52,6 +53,7 @@ export async function createServer(options?: {
   await app.register(noteRoutes);
   await app.register(configRoutes);
   await app.register(relationRoutes);
+  await app.register(memoryRoutes);
 
   // Serve frontend in production
   // Try multiple possible paths (source layout vs compiled layout)

server/src/routes/conversations.ts

@@ -15,7 +15,7 @@ export async function conversationRoutes(app: FastifyInstance) {
     } = req.query as Record<string, string>;
 
     const db = getDatabase();
-    const conditions: string[] = [];
+    const conditions: string[] = ["c.source != 'chatcrystal-memory'"];
     const params: (string | number)[] = [];
 
     if (source) {
@@ -109,4 +109,3 @@ export async function conversationRoutes(app: FastifyInstance) {
     return { success: true };
   });
 }
-

server/src/routes/memory.test.ts

@@ -0,0 +1,67 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import Fastify from 'fastify';
+import { memoryRoutes } from './memory.js';
+
+test('memory recall returns 400 for validation failures', async () => {
+  const app = Fastify();
+  await app.register(memoryRoutes);
+
+  const response = await app.inject({
+    method: 'POST',
+    url: '/api/memory/recall',
+    payload: {
+      mode: 'task',
+      task: {
+        task_kind: 'debug',
+      },
+    },
+  });
+
+  assert.equal(response.statusCode, 400);
+  await app.close();
+});
+
+test('memory recall returns 500 for internal failures after validation succeeds', async () => {
+  const app = Fastify();
+  await app.register(memoryRoutes);
+
+  const response = await app.inject({
+    method: 'POST',
+    url: '/api/memory/recall',
+    payload: {
+      mode: 'task',
+      task: {
+        goal: 'Fix flaky timeout',
+        task_kind: 'debug',
+      },
+    },
+  });
+
+  assert.equal(response.statusCode, 500);
+  await app.close();
+});
+
+test('memory writeback returns 500 for internal failures after validation succeeds', async () => {
+  const app = Fastify();
+  await app.register(memoryRoutes);
+
+  const response = await app.inject({
+    method: 'POST',
+    url: '/api/memory/writeback',
+    payload: {
+      mode: 'manual',
+      task: {
+        goal: 'Capture a reusable fix',
+        task_kind: 'debug',
+      },
+      memory: {
+        summary: 'Await readiness before requests.',
+        outcome_type: 'fix',
+      },
+    },
+  });
+
+  assert.equal(response.statusCode, 500);
+  await app.close();
+});

server/src/routes/memory.ts

@@ -0,0 +1,38 @@
+import type { FastifyInstance } from 'fastify';
+import { ZodError } from 'zod';
+import { recallForTask } from '../services/memory/recall.js';
+import { writeTaskMemory } from '../services/memory/writeback.js';
+
+export async function memoryRoutes(app: FastifyInstance) {
+  app.post('/api/memory/recall', async (req, reply) => {
+    try {
+      const data = await recallForTask(req.body);
+      return { success: true, data };
+    } catch (error) {
+      reply.status(error instanceof ZodError ? 400 : 500);
+      return {
+        success: false,
+        error:
+          error instanceof Error
+            ? error.message
+            : 'Invalid recall request',
+      };
+    }
+  });
+
+  app.post('/api/memory/writeback', async (req, reply) => {
+    try {
+      const data = await writeTaskMemory(req.body);
+      return { success: true, data };
+    } catch (error) {
+      reply.status(error instanceof ZodError ? 400 : 500);
+      return {
+        success: false,
+        error:
+          error instanceof Error
+            ? error.message
+            : 'Invalid writeback request',
+      };
+    }
+  });
+}

server/src/routes/notes.ts

@@ -10,6 +10,8 @@ function hydrateNote(row: Record<string, unknown>): Record<string, unknown> {
     ...row,
     key_conclusions: row.key_conclusions ? JSON.parse(row.key_conclusions as string) : [],
     code_snippets: row.code_snippets ? JSON.parse(row.code_snippets as string) : [],
+    error_signatures: row.error_signatures ? JSON.parse(row.error_signatures as string) : [],
+    files_touched: row.files_touched ? JSON.parse(row.files_touched as string) : [],
     tags: row.tags_csv ? (row.tags_csv as string).split(',') : [],
     is_edited: Boolean(row.is_edited),
   };