feat: Phase 1+2 Background Autonomy — FS Watcher + Continuous Learning Pipeline

.planning/phases/01-fs-watcher/01-01-PLAN.md

@@ -0,0 +1,159 @@
+---
+phase: 01-fs-watcher
+plan: 01
+type: execute
+wave: 1
+depends_on: []
+files_modified:
+  - universal-refiner/src/watcher/file-watcher.ts
+  - universal-refiner/src/watcher/index.ts
+  - universal-refiner/tests/file-watcher.test.ts
+  - universal-refiner/src/index.ts
+autonomous: true
+requirements:
+  - AUTO-01
+  - AUTO-02
+must_haves:
+  truths:
+    - "FileWatcher emits a 'change' event when a watched .ts file is written"
+    - "FileWatcher emits an 'add' event when a new .ts file appears"
+    - "FileWatcher does not emit for files inside node_modules"
+    - "FileWatcher does not emit for *.log or *.tmp files"
+    - "FileWatcher.stop() prevents any further events"
+    - "Detected changes are logged via RuntimeLogger on server startup"
+  artifacts:
+    - path: "universal-refiner/src/watcher/file-watcher.ts"
+      provides: "FileWatcher class with start/stop/on('change') interface"
+      exports: ["FileWatcher", "FileChangeEvent", "FileEventKind"]
+    - path: "universal-refiner/src/watcher/index.ts"
+      provides: "Re-exports for the watcher module"
+    - path: "universal-refiner/tests/file-watcher.test.ts"
+      provides: "5 Vitest tests covering AUTO-01 and AUTO-02"
+  key_links:
+    - from: "universal-refiner/src/index.ts"
+      to: "universal-refiner/src/watcher/index.ts"
+      via: "import FileWatcher, call start() at server init"
+---
+
+<objective>
+Implement a real-time file system watcher (Phase 1) for the universal-refiner MCP server.
+
+Purpose: Satisfy AUTO-01 (detect meaningful file save events) and AUTO-02 (filter noise paths) as the foundation for the Background Autonomy milestone.
+
+Output:
+- src/watcher/file-watcher.ts — FileWatcher class wrapping chokidar v5
+- src/watcher/index.ts — re-exports
+- tests/file-watcher.test.ts — 5 passing Vitest tests
+- src/index.ts updated to start watcher on server init
+</objective>
+
+<execution_context>
+chokidar v5.0.0 is already in dependencies. No new packages needed.
+Uses RuntimeLogger (stderr, JSON-RPC safe) for all output.
+</execution_context>
+
+<context>
+@.planning/ROADMAP.md
+@.planning/REQUIREMENTS.md
+@universal-refiner/src/core/logger.ts
+@universal-refiner/src/index.ts
+</context>
+
+<tasks>
+
+<task type="auto">
+  <name>Task 1: Create FileWatcher module (AUTO-01, AUTO-02)</name>
+  <files>
+    universal-refiner/src/watcher/file-watcher.ts
+    universal-refiner/src/watcher/index.ts
+  </files>
+  <action>
+    FileWatcher extends EventEmitter. Constructor takes rootPath: string.
+    start(): watches rootPath via chokidar.watch() with ignored: CHOKIDAR_IGNORE patterns.
+    stop(): closes the chokidar watcher, nulls inner reference.
+    emitChange() applies two-layer filter before emitting:
+      1. Path segment check: reject paths containing /node_modules/, /dist/, /.git/, /coverage/
+      2. Extension check: only emit for .ts, .js, .md, .txt, .prompt
+      3. Suffix noise check: reject .log, .tmp
+    Emits: { path: string, event: 'add'|'change'|'unlink', timestamp: Date }
+    Logs start/stop and per-event debug via RuntimeLogger.
+    index.ts re-exports FileWatcher, FileChangeEvent, FileEventKind.
+  </action>
+  <verify>npm run build -- succeeds with zero type errors</verify>
+  <done>Both files exist, TypeScript compiles clean, exports are correct.</done>
+</task>
+
+<task type="auto" tdd="true">
+  <name>Task 2: Write Vitest tests (AUTO-01, AUTO-02)</name>
+  <files>universal-refiner/tests/file-watcher.test.ts</files>
+  <behavior>
+    - Test: write to existing .ts file in tmp dir -> 'change' event emitted with correct path and Date timestamp
+    - Test: write new .ts file -> 'add' event emitted
+    - Test: write .ts file inside node_modules subdirectory -> no event emitted (AUTO-02)
+    - Test: write .log file -> no event emitted (AUTO-02)
+    - Test: stop() called -> subsequent file write produces no events
+  </behavior>
+  <action>
+    Use vitest describe/it/expect. beforeEach creates a unique tmp dir via fs.mkdtempSync.
+    afterEach calls watcher.stop() and fs.rmSync.
+    Use a polling waitFor() helper with 6000ms timeout for positive assertions.
+    Allow 1500ms settle time after watcher.start() before writing files (Windows FS listener warm-up).
+    Set per-test timeout to 15_000.
+  </action>
+  <verify>npm test -- shows 5/5 file-watcher tests passing</verify>
+  <done>All 48 total tests pass including the 5 new watcher tests.</done>
+</task>
+
+<task type="auto">
+  <name>Task 3: Wire FileWatcher into server entry point</name>
+  <files>universal-refiner/src/index.ts</files>
+  <action>
+    Import FileWatcher from "./watcher/index.js".
+    After CommandCenterDashboard.start() and before runBackgroundTasks():
+      const fileWatcher = new FileWatcher(rootPath);
+      fileWatcher.on('change', (evt) => {
+        RuntimeLogger.info(`[FS] ${evt.event}: ${evt.path}`);
+        CommandCenterDashboard.log(`[FS] ${evt.event}: ${path.relative(rootPath, evt.path)}`);
+      });
+      fileWatcher.start();
+    BackgroundAutonomyService is left intact — FileWatcher is additive.
+  </action>
+  <verify>npm run build succeeds; server starts without error</verify>
+  <done>File watcher starts automatically when the MCP server initialises.</done>
+</task>
+
+</tasks>
+
+<threat_model>
+## Trust Boundaries
+
+| Boundary | Description |
+|----------|-------------|
+| FS path → emitChange | File paths from chokidar are OS-provided and not sanitised before logging |
+
+## STRIDE Threat Register
+
+| Threat ID | Category | Component | Disposition | Mitigation Plan |
+|-----------|----------|-----------|-------------|-----------------|
+| T-01-01 | Information Disclosure | RuntimeLogger path output | accept | Logs go to stderr/runtime.log, not stdout (JSON-RPC channel). No PII in paths. |
+| T-01-02 | Denial of Service | Rapid file changes flood emitChange | mitigate | awaitWriteFinish debounce (100ms stability) prevents event storms. |
+| T-01-03 | Tampering | Malicious path with path-traversal characters | accept | FileWatcher is read-only (no FS writes). Path is logged, not executed. |
+</threat_model>
+
+<verification>
+- npm run build -- zero TypeScript errors
+- npm test -- 48/48 tests pass (5 new FileWatcher tests)
+- Server starts and logs "[FileWatcher] Starting file system watcher" on init
+</verification>
+
+<success_criteria>
+1. Writing a .ts file in the watched directory emits a change event within 3 seconds.
+2. Files under node_modules, dist, .git, coverage are never emitted.
+3. .log and .tmp files are never emitted.
+4. stop() terminates all event delivery immediately.
+5. Build and full test suite remain green.
+</success_criteria>
+
+<output>
+Create .planning/phases/01-fs-watcher/01-01-SUMMARY.md after execution.
+</output>

universal-refiner/src/index.ts

@@ -1,9 +1,11 @@
-#!/usr/bin/env node
+#!/usr/bin/env node
 import { PromptRefinerServer } from "./core/server.js";
 import { CommandCenterDashboard } from "./core/dashboard.js";
 import { CommitIngester } from "./history/commit-ingest.js";
 import { CorrelationEngine } from "./history/correlation-engine.js";
 import { LessonExtractor } from "./history/lesson-extractor.js";
+import { FileWatcher } from "./watcher/index.js";
+import { RuntimeLogger } from "./core/logger.js";
 import * as path from "path";
 
 // Start the Web Dashboard in the background
@@ -13,12 +15,21 @@ CommandCenterDashboard.start(port, rootPath);
 
 const server = new PromptRefinerServer(rootPath);
 
+// Phase 1 (AUTO-01, AUTO-02): Real-time file system watcher
+// Starts watching for meaningful source/prompt file changes and logs them.
+const fileWatcher = new FileWatcher(rootPath);
+fileWatcher.on("change", (evt) => {
+  RuntimeLogger.info(`[FS] ${evt.event}: ${evt.path}`);
+  CommandCenterDashboard.log(`[FS] ${evt.event}: ${path.relative(rootPath, evt.path)}`);
+});
+fileWatcher.start();
+
 // Initial background ingestion and correlation
 async function runBackgroundTasks() {
   await CommitIngester.ingestLatest(rootPath);
   const correlation = new CorrelationEngine();
   await correlation.correlateAll();
-  
+
   const extractor = new LessonExtractor((task, prompt, tokens) => server.requestModelText(task, prompt, tokens));
   await extractor.extractNewLessons();
 }

universal-refiner/src/watcher/file-watcher.ts

@@ -0,0 +1,159 @@
+import { EventEmitter } from "node:events";
+import { watch as chokidarWatch, FSWatcher } from "chokidar";
+import { RuntimeLogger } from "../core/logger.js";
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+export type FileEventKind = "add" | "change" | "unlink";
+
+export interface FileChangeEvent {
+  path: string;
+  event: FileEventKind;
+  timestamp: Date;
+}
+
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+
+/** File extensions considered "meaningful" for AUTO-01 */
+export const MEANINGFUL_EXTENSIONS = new Set([".ts", ".js", ".md", ".txt", ".prompt"]);
+
+/**
+ * Path segments that mark noise directories (AUTO-02).
+ * Used as a secondary in-process guard after chokidar's ignore patterns.
+ */
+export const NOISE_PATH_SEGMENTS = ["node_modules", "dist", ".git", "coverage"];
+
+/** File suffixes that mark noise files (AUTO-02). */
+export const NOISE_SUFFIXES = [".log", ".tmp"];
+
+/** Paths and patterns passed to chokidar `ignored` option (AUTO-02). */
+const CHOKIDAR_IGNORE: (string | RegExp)[] = [
+  "**/node_modules/**",
+  "**/dist/**",
+  "**/.git/**",
+  "**/*.log",
+  "**/*.tmp",
+  "**/coverage/**",
+];
+
+// ---------------------------------------------------------------------------
+// FileWatcher
+// ---------------------------------------------------------------------------
+
+/**
+ * FileWatcher wraps chokidar with a focused, typed EventEmitter interface.
+ *
+ * Responsibilities:
+ *  - Detect meaningful source/prompt file changes (AUTO-01)
+ *  - Filter transient/noise paths before emitting (AUTO-02)
+ *
+ * Usage:
+ *   const watcher = new FileWatcher('/path/to/project');
+ *   watcher.on('change', (evt) => console.log(evt));
+ *   watcher.start();
+ *   // later
+ *   await watcher.stop();
+ */
+export class FileWatcher extends EventEmitter {
+  private readonly rootPath: string;
+  private inner: FSWatcher | null = null;
+
+  constructor(rootPath: string) {
+    super();
+    this.rootPath = rootPath;
+  }
+
+  // ------------------------------------------------------------------
+  // Public API
+  // ------------------------------------------------------------------
+
+  /** Start watching. Idempotent — calling twice is a no-op. */
+  public start(): void {
+    if (this.inner) {
+      return;
+    }
+
+    RuntimeLogger.info("[FileWatcher] Starting file system watcher", {
+      rootPath: this.rootPath,
+    });
+
+    this.inner = chokidarWatch(this.rootPath, {
+      ignored: CHOKIDAR_IGNORE,
+      persistent: true,
+      ignoreInitial: true,
+      awaitWriteFinish: {
+        stabilityThreshold: 100,
+        pollInterval: 50,
+      },
+    });
+
+    this.inner.on("add", (filePath) => this.emitChange("add", filePath));
+    this.inner.on("change", (filePath) => this.emitChange("change", filePath));
+    this.inner.on("unlink", (filePath) => this.emitChange("unlink", filePath));
+    this.inner.on("error", (err: unknown) => {
+      RuntimeLogger.error("[FileWatcher] Watcher error", err);
+      this.emit("error", err instanceof Error ? err : new Error(String(err)));
+    });
+  }
+
+  /** Stop watching and release all resources. */
+  public async stop(): Promise<void> {
+    if (!this.inner) {
+      return;
+    }
+    await this.inner.close();
+    this.inner = null;
+    RuntimeLogger.info("[FileWatcher] Stopped file system watcher");
+  }
+
+  // ------------------------------------------------------------------
+  // Internal helpers
+  // ------------------------------------------------------------------
+
+  /**
+   * Apply both AUTO-01 and AUTO-02 filters before emitting.
+   *
+   * Two-layer defence:
+   *  1. chokidar `ignored` patterns (coarse, path-glob based)
+   *  2. In-process extension + segment checks (precise, cross-platform)
+   *
+   * The in-process layer catches edge cases where chokidar's glob ignore
+   * may lag (e.g. directories created before the watcher starts on Windows).
+   */
+  private emitChange(kind: FileEventKind, filePath: string): void {
+    // Normalise separators for consistent matching on Windows
+    const normalised = filePath.replace(/\\/g, "/");
+
+    // AUTO-02: segment-level noise filter
+    for (const seg of NOISE_PATH_SEGMENTS) {
+      if (normalised.includes(`/${seg}/`) || normalised.includes(`/${seg}`)) {
+        return;
+      }
+    }
+
+    // AUTO-01: extension filter — only emit for meaningful file types
+    const dotIdx = normalised.lastIndexOf(".");
+    const ext = dotIdx >= 0 ? normalised.slice(dotIdx).toLowerCase() : "";
+
+    // AUTO-02: suffix-level noise filter (e.g. .log, .tmp)
+    if (NOISE_SUFFIXES.includes(ext)) {
+      return;
+    }
+
+    if (!MEANINGFUL_EXTENSIONS.has(ext)) {
+      return;
+    }
+
+    const evt: FileChangeEvent = {
+      path: filePath,
+      event: kind,
+      timestamp: new Date(),
+    };
+    RuntimeLogger.debug(`[FileWatcher] ${kind}: ${filePath}`);
+    this.emit("change", evt);
+  }
+}

universal-refiner/src/watcher/index.ts

@@ -0,0 +1,2 @@
+export { FileWatcher } from "./file-watcher.js";
+export type { FileChangeEvent, FileEventKind } from "./file-watcher.js";