Add support for -i flags in Discovery mode with ReadOnlyLayeredStore

richhankins · richhankins · commit dcf23860e25b · 2026-02-06T13:58:06.000-08:00
- Create ReadOnlyLayeredStore to merge local (FilesystemStore) and remote (CompositeStoreReader) indexes
- Update cmd-mcp.ts to use ReadOnlyLayeredStore when both --discovery and -i flags are present
- Add comprehensive tests for ReadOnlyLayeredStore (12 tests)
- Export ReadOnlyLayeredStore from stores/index.ts

Behavior:
- ctxc mcp stdio --discovery -i s3://bucket/shared-index: Merges local + remote indexes
- ctxc mcp stdio -i pytorch -i react: Fixed mode (no discovery)
- ctxc mcp stdio --discovery: Discovery mode with local indexes only

All tests pass (175 passed, 24 skipped)

Agent-Id: agent-5834efcf-335b-415a-b8a9-19ed489133ed
diff --git a/src/bin/cmd-mcp.ts b/src/bin/cmd-mcp.ts
@@ -7,6 +7,7 @@ import { FilesystemStore } from "../stores/filesystem.js";
 import { runMCPServer } from "../clients/mcp-server.js";
 import { parseIndexSpecs } from "../stores/index-spec.js";
 import { CompositeStoreReader } from "../stores/composite.js";
+import { ReadOnlyLayeredStore } from "../stores/read-only-layered-store.js";
 
 // stdio subcommand (stdio-based MCP server for local clients like Claude Desktop)
 const stdioCommand = new Command("stdio")
@@ -20,28 +21,34 @@ const stdioCommand = new Command("stdio")
   .action(async (options) => {
     try {
       const indexSpecs: string[] | undefined = options.index;
-      const discovery = options.discovery || !indexSpecs || indexSpecs.length === 0;
+      const discoveryFlag = options.discovery;
 
       let store;
       let indexNames: string[] | undefined;
+      let discovery: boolean;
 
-      if (indexSpecs && indexSpecs.length > 0) {
-        // Parse index specs
+      if (discoveryFlag && indexSpecs && indexSpecs.length > 0) {
+        // Discovery mode WITH remote indexes: merge local + remote
         const specs = parseIndexSpecs(indexSpecs);
-        indexNames = specs.map((s) => s.displayName);
-
+        const remoteStore = await CompositeStoreReader.fromSpecs(specs);
+        const localStore = new FilesystemStore();
+        store = new ReadOnlyLayeredStore(localStore, remoteStore);
+        indexNames = undefined; // Discovery mode: no fixed list
+        discovery = true;
+      } else if (indexSpecs && indexSpecs.length > 0) {
         // Fixed mode: use read-only CompositeStoreReader
+        const specs = parseIndexSpecs(indexSpecs);
+        indexNames = specs.map((s) => s.displayName);
         store = await CompositeStoreReader.fromSpecs(specs);
+        discovery = false;
       } else {
-        // No --index: use FilesystemStore (discovery mode)
+        // Discovery mode only: use FilesystemStore
         store = new FilesystemStore();
-        // Discovery mode: server can start with zero indexes
-        // Use list_indexes to see available indexes, manage via CLI
         indexNames = undefined;
+        discovery = true;
       }
 
       // Start MCP server (writes to stdout, reads from stdin)
-      // discovery: true when no -i flags (discovery mode), false when -i flags provided (fixed mode)
       await runMCPServer({
         store,
         indexNames,
@@ -75,24 +82,31 @@ const httpCommand = new Command("http")
   .action(async (options) => {
     try {
       const indexSpecs: string[] | undefined = options.index;
-      const discovery = options.discovery || !indexSpecs || indexSpecs.length === 0;
+      const discoveryFlag = options.discovery;
 
       let store;
       let indexNames: string[] | undefined;
+      let discovery: boolean;
 
-      if (indexSpecs && indexSpecs.length > 0) {
-        // Parse index specs
+      if (discoveryFlag && indexSpecs && indexSpecs.length > 0) {
+        // Discovery mode WITH remote indexes: merge local + remote
         const specs = parseIndexSpecs(indexSpecs);
-        indexNames = specs.map((s) => s.displayName);
-
+        const remoteStore = await CompositeStoreReader.fromSpecs(specs);
+        const localStore = new FilesystemStore();
+        store = new ReadOnlyLayeredStore(localStore, remoteStore);
+        indexNames = undefined; // Discovery mode: no fixed list
+        discovery = true;
+      } else if (indexSpecs && indexSpecs.length > 0) {
         // Fixed mode: use read-only CompositeStoreReader
+        const specs = parseIndexSpecs(indexSpecs);
+        indexNames = specs.map((s) => s.displayName);
         store = await CompositeStoreReader.fromSpecs(specs);
+        discovery = false;
       } else {
-        // No --index: use FilesystemStore (discovery mode)
+        // Discovery mode only: use FilesystemStore
         store = new FilesystemStore();
-        // Discovery mode: server can start with zero indexes
-        // Use list_indexes to see available indexes, manage via CLI
         indexNames = undefined;
+        discovery = true;
       }
 
       // Parse CORS option
@@ -108,7 +122,6 @@ const httpCommand = new Command("http")
       const apiKey = options.apiKey ?? process.env.MCP_API_KEY;
 
       // Start HTTP server
-      // discovery: true when no -i flags (discovery mode), false when -i flags provided (fixed mode)
       const { runMCPHttpServer } = await import("../clients/mcp-http-server.js");
       const server = await runMCPHttpServer({
         store,
diff --git a/src/stores/index.ts b/src/stores/index.ts
@@ -10,6 +10,7 @@ export type { MemoryStoreConfig } from "./memory.js";
 export { S3Store } from "./s3.js";
 export type { S3StoreConfig } from "./s3.js";
 export { CompositeStoreReader } from "./composite.js";
+export { ReadOnlyLayeredStore } from "./read-only-layered-store.js";
 export { parseIndexSpec, parseIndexSpecs } from "./index-spec.js";
 export type { IndexSpec } from "./index-spec.js";
 
diff --git a/src/stores/read-only-layered-store.test.ts b/src/stores/read-only-layered-store.test.ts
@@ -0,0 +1,188 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { ReadOnlyLayeredStore } from "./read-only-layered-store.js";
+import { MemoryStore } from "./memory.js";
+import type { IndexState, IndexStateSearchOnly } from "../core/types.js";
+
+function createMockState(): { full: IndexState; search: IndexStateSearchOnly } {
+  const source = {
+    type: "github" as const,
+    config: { owner: "test-owner", repo: "test-repo" },
+    syncedAt: new Date().toISOString(),
+  };
+  return {
+    full: {
+      version: 1,
+      contextState: {
+        mode: "full" as const,
+        checkpointId: "test-checkpoint-123",
+        addedBlobs: ["blob-1"],
+        deletedBlobs: [],
+        blobs: [["blob-1", "src/file1.ts"]],
+      },
+      source,
+    },
+    search: {
+      version: 1,
+      contextState: {
+        mode: "search-only" as const,
+        checkpointId: "test-checkpoint-123",
+        addedBlobs: ["blob-1"],
+        deletedBlobs: [],
+      },
+      source,
+    },
+  };
+}
+
+describe("ReadOnlyLayeredStore", () => {
+  let primary: MemoryStore;
+  let remote: MemoryStore;
+  let layered: ReadOnlyLayeredStore;
+
+  beforeEach(() => {
+    primary = new MemoryStore();
+    remote = new MemoryStore();
+    layered = new ReadOnlyLayeredStore(primary, remote);
+  });
+
+  describe("list", () => {
+    it("returns merged and deduplicated list from both stores", async () => {
+      const { full, search } = createMockState();
+
+      // Add to primary
+      await primary.save("local-a", full, search);
+      await primary.save("local-b", full, search);
+
+      // Add to remote
+      await remote.save("remote-a", full, search);
+      await remote.save("remote-b", full, search);
+
+      const list = await layered.list();
+
+      expect(list).toContain("local-a");
+      expect(list).toContain("local-b");
+      expect(list).toContain("remote-a");
+      expect(list).toContain("remote-b");
+      expect(list.length).toBe(4);
+    });
+
+    it("deduplicates when same index exists in both stores", async () => {
+      const { full, search } = createMockState();
+
+      // Add same index to both
+      await primary.save("shared", full, search);
+      await remote.save("shared", full, search);
+
+      const list = await layered.list();
+
+      expect(list).toContain("shared");
+      expect(list.length).toBe(1);
+    });
+
+    it("returns sorted list", async () => {
+      const { full, search } = createMockState();
+
+      await primary.save("zebra", full, search);
+      await primary.save("apple", full, search);
+      await remote.save("mango", full, search);
+
+      const list = await layered.list();
+
+      expect(list).toEqual(["apple", "mango", "zebra"]);
+    });
+
+    it("returns empty list when both stores are empty", async () => {
+      const list = await layered.list();
+      expect(list).toEqual([]);
+    });
+  });
+
+  describe("loadSearch", () => {
+    it("returns from primary if exists", async () => {
+      const { full, search } = createMockState();
+      await primary.save("test", full, search);
+
+      const result = await layered.loadSearch("test");
+
+      expect(result).toEqual(search);
+    });
+
+    it("falls back to remote if not in primary", async () => {
+      const { full, search } = createMockState();
+      await remote.save("test", full, search);
+
+      const result = await layered.loadSearch("test");
+
+      expect(result).toEqual(search);
+    });
+
+    it("prefers primary over remote when both exist", async () => {
+      const { full, search } = createMockState();
+      const primarySearch = {
+        ...search,
+        contextState: { ...search.contextState, checkpointId: "primary" },
+      };
+      const remoteSearch = {
+        ...search,
+        contextState: { ...search.contextState, checkpointId: "remote" },
+      };
+
+      await primary.save("test", full, primarySearch);
+      await remote.save("test", full, remoteSearch);
+
+      const result = await layered.loadSearch("test");
+
+      expect(result?.contextState.checkpointId).toBe("primary");
+    });
+
+    it("returns null if not found in either store", async () => {
+      const result = await layered.loadSearch("nonexistent");
+      expect(result).toBeNull();
+    });
+  });
+
+  describe("loadState", () => {
+    it("returns from primary if exists", async () => {
+      const { full, search } = createMockState();
+      await primary.save("test", full, search);
+
+      const result = await layered.loadState("test");
+
+      expect(result).toEqual(full);
+    });
+
+    it("falls back to remote if not in primary", async () => {
+      const { full, search } = createMockState();
+      await remote.save("test", full, search);
+
+      const result = await layered.loadState("test");
+
+      expect(result).toEqual(full);
+    });
+
+    it("prefers primary over remote when both exist", async () => {
+      const { full, search } = createMockState();
+      const primaryFull = {
+        ...full,
+        contextState: { ...full.contextState, checkpointId: "primary" },
+      };
+      const remoteFull = {
+        ...full,
+        contextState: { ...full.contextState, checkpointId: "remote" },
+      };
+
+      await primary.save("test", primaryFull, search);
+      await remote.save("test", remoteFull, search);
+
+      const result = await layered.loadState("test");
+
+      expect(result?.contextState.checkpointId).toBe("primary");
+    });
+
+    it("returns null if not found in either store", async () => {
+      const result = await layered.loadState("nonexistent");
+      expect(result).toBeNull();
+    });
+  });
+});
+
diff --git a/src/stores/read-only-layered-store.ts b/src/stores/read-only-layered-store.ts
@@ -0,0 +1,82 @@
+/**
+ * Read-only layered store that merges a primary store with a remote store.
+ *
+ * Used in discovery mode with remote indexes:
+ * - Primary store: FilesystemStore (local indexes)
+ * - Remote store: CompositeStoreReader (remote indexes)
+ *
+ * Behavior:
+ * - list(): Merge both lists, deduplicated, sorted
+ * - loadState(name): Try primary first, then remote
+ * - loadSearch(name): Try primary first, then remote
+ * - No save/delete methods (read-only)
+ *
+ * @module stores/read-only-layered-store
+ */
+
+import type { IndexStoreReader } from "./types.js";
+import type { IndexState, IndexStateSearchOnly } from "../core/types.js";
+
+/**
+ * Read-only layered store that combines a primary and remote store.
+ *
+ * Useful for discovery mode where users can:
+ * - Manage local indexes via CLI (stored in FilesystemStore)
+ * - Reference remote indexes via -i flags (stored in CompositeStoreReader)
+ *
+ * @example
+ * ```typescript
+ * const primary = new FilesystemStore();
+ * const remote = await CompositeStoreReader.fromSpecs([
+ *   { type: "s3", value: "s3://bucket/shared-index", displayName: "shared" }
+ * ]);
+ * const layered = new ReadOnlyLayeredStore(primary, remote);
+ *
+ * // Lists both local and remote indexes
+ * const allIndexes = await layered.list();
+ *
+ * // Tries primary first, then remote
+ * const state = await layered.loadSearch("my-index");
+ * ```
+ */
+export class ReadOnlyLayeredStore implements IndexStoreReader {
+  constructor(
+    private primary: IndexStoreReader,
+    private remote: IndexStoreReader
+  ) {}
+
+  async loadState(key: string): Promise<IndexState | null> {
+    // Try primary first
+    const primaryState = await this.primary.loadState(key);
+    if (primaryState !== null) {
+      return primaryState;
+    }
+    // Fall back to remote
+    return this.remote.loadState(key);
+  }
+
+  async loadSearch(key: string): Promise<IndexStateSearchOnly | null> {
+    // Try primary first
+    const primarySearch = await this.primary.loadSearch(key);
+    if (primarySearch !== null) {
+      return primarySearch;
+    }
+    // Fall back to remote
+    return this.remote.loadSearch(key);
+  }
+
+  async list(): Promise<string[]> {
+    // Get both lists
+    const [primaryList, remoteList] = await Promise.all([
+      this.primary.list(),
+      this.remote.list(),
+    ]);
+
+    // Merge and deduplicate
+    const merged = new Set([...primaryList, ...remoteList]);
+
+    // Return sorted array
+    return Array.from(merged).sort();
+  }
+}
+
