diff --git a/CHANGELOG.md b/CHANGELOG.md index bb03d4d..604d150 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,15 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release ## [Unreleased] +### Added + +- **Per-source and per-namespace private config:** optional `description` (source-level) and `namespaces` map (`description` + declared `metadata_schema`) in JSON config files (`PINECONE_CONFIG_FILE` only — not inline `PINECONE_SOURCES`). Declared schemas skip live sampling; stale declared namespaces surface as `config_warnings` in `list_namespaces`. See [CONFIGURATION.md](docs/CONFIGURATION.md#json-config-file). +- **`list_namespaces`:** optional per-namespace `schema_source` (`declared` | `sampled`), optional per-namespace `description` (from private config), and top-level `config_warnings` when private config declarations do not match live Pinecone data. + ### Changed +- **Breaking (`list_sources`):** response shape `sources: string[]` → `sources: { name, description? }[]`. Migration: [MIGRATION.md § Unreleased list_sources](docs/MIGRATION.md#unreleased-list_sources-response-shape). +- **Breaking (library API):** `PineconeClient.listNamespacesWithMetadata()` now returns `{ namespaces, warnings }` instead of a bare array. Migration: [MIGRATION.md § Unreleased PineconeClient.listNamespacesWithMetadata](docs/MIGRATION.md#unreleased-pineconeclientlistnamespaceswithmetadata-return-shape). - **Instructions:** Trimmed operator/install/deploy content (env-var setup, misconfiguration note, Alliance CLI index/rerank defaults, stderr logging config) from `CORE_SERVER_INSTRUCTIONS` and `ALLIANCE_INSTRUCTIONS_APPENDIX` — reduces per-session token cost; no behavior change. Replaced colliding Alliance appendix steps 4–5 with unnumbered "Manual Alliance flow" bullets (includes `PINECONE_DISABLE_SUGGEST_FLOW=true` escape clause). Full detail remains in [docs/CONFIGURATION.md](docs/CONFIGURATION.md). ## [0.4.0] - 2026-06-24 diff --git a/benchmarks/latency.ts b/benchmarks/latency.ts index c012581..ddeda7a 100644 --- a/benchmarks/latency.ts +++ b/benchmarks/latency.ts @@ -233,7 +233,10 @@ function createBenchPineconeMock(): PineconeClient { return { count: 42, truncated: false }; }, async listNamespacesWithMetadata() { - return namespaces; + return { + namespaces: namespaces.map((n) => ({ ...n, schema_source: 'sampled' as const })), + warnings: [], + }; }, async listNamespacesFromKeywordIndex() { return namespaces.map((n) => ({ namespace: n.namespace, recordCount: n.recordCount })); diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 650fc53..a107c22 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -75,6 +75,17 @@ Set `PINECONE_CONFIG_FILE` (or `--config-file`) to a path such as [examples/mult Values support `${ENV_VAR}` indirection (resolved at startup). Per-source `sparseIndexName` and `rerankModel` are optional; Alliance defaults apply when omitted. +Optional **config-file-only** fields (not supported in inline `PINECONE_SOURCES`): + +| Field | Scope | Purpose | +| ----- | ----- | ------- | +| `description` | Per source | Short corpus/content hint surfaced via `list_sources` (helps LLM routing across sources or MCP entries) | +| `namespaces` | Per source | Map of namespace name → `{ description?, metadata_schema? }`; per-namespace `description` is surfaced via `list_namespaces` on matching live rows | + +`metadata_schema` is a flat `fieldName → type` map (same vocabulary as `list_namespaces` → `metadata_fields`, e.g. `"title": "string"`). When declared for a live namespace, the server **skips live sampling** for that namespace and trusts the declared schema until the config file changes. Namespaces declared in config but absent from Pinecone produce a non-fatal `config_warnings` entry in `list_namespaces` (never a startup failure). + +**Never** commit real corpus descriptions, namespace names, or internal field names to the open-source repo — use generic placeholders in examples only. Real values belong in staff-machine private config files per [Deployment profiles](#deployment-profiles). See [SECURITY.md](./SECURITY.md). + ### MCP tools and routing | Tool | `source` parameter | @@ -131,7 +142,7 @@ Multi-source mode supports two operational profiles. **Never** ship a merged int } ``` -Prefer `PINECONE_CONFIG_FILE` with `${ENV_VAR}` indirection over inline API keys in `PINECONE_SOURCES`. See [SECURITY.md](./SECURITY.md). +Prefer `PINECONE_CONFIG_FILE` with `${ENV_VAR}` indirection over inline API keys in `PINECONE_SOURCES`. For internal deployments, add optional `description` and `namespaces` declarations in the JSON config file on staff machines only — never in public examples or committed constants. See [SECURITY.md](./SECURITY.md). ### Architecture decision diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 4799680..5f65e3e 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -6,6 +6,56 @@ This guide is for **library and MCP client authors** upgrading from earlier **0. Under [semver 0.y.z](https://semver.org/spec/v2.0.0.html#spec-item-4), **0.1.x → 0.2.0 is a breaking minor** — pin `@0.2.0` only after reading this guide. +## Unreleased: `list_sources` response shape + +**Who is affected:** MCP clients parsing `list_sources` success JSON in multi-source mode. + +**Before:** + +```json +{ + "status": "success", + "sources": ["api_key_1", "api_key_2"], + "default": "api_key_1" +} +``` + +**After:** + +```json +{ + "status": "success", + "sources": [ + { "name": "api_key_1", "description": "Optional corpus hint from private config" }, + { "name": "api_key_2" } + ], + "default": "api_key_1" +} +``` + +Read `sources[].name` instead of treating `sources` as `string[]`. `description` is omitted when not configured. + +## Unreleased: `PineconeClient.listNamespacesWithMetadata` return shape + +**Who is affected:** Direct library consumers of `PineconeClient` (not MCP tool clients — `list_namespaces` tool response shape is additive only). + +**Before:** + +```ts +const rows = await client.listNamespacesWithMetadata(); +// rows: Array<{ namespace, recordCount, metadata }> +``` + +**After:** + +```ts +const { namespaces, warnings } = await client.listNamespacesWithMetadata(declaredSchemas); +// namespaces: Array<{ namespace, recordCount, metadata, schema_source }> +// warnings: string[] — e.g. stale declared namespace not found live +``` + +Optional `declaredSchemas` argument skips live sampling for namespaces with a declared schema. + ## Unreleased: Stable vs experimental response fields **Rationale:** Tool success payloads mixed stable contract fields with experimental diagnostics (`degraded`, `decision_trace`, etc.) at the top level. Experimental fields are now nested under `experimental` so consumers know which fields are safe across minor version bumps. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index bbf1b51..dcd338f 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -23,6 +23,10 @@ Tool responses returned to MCP clients (and LLM consumers) are sanitized in `src This covers tool error payloads, hybrid degradation reasons, and SDK error text surfaced in DEBUG log mode — not only stderr logs. +## Private config content (descriptions and schemas) + +Per-source `description` and per-namespace `namespaces` (including namespace names and `metadata_schema` field names) loaded from `PINECONE_CONFIG_FILE` are **High**-risk if committed to the open-source repo or shared through public distribution channels — they can disclose what private corpora contain and which internal metadata fields exist. Keep real values on staff machines only; use generic placeholders in examples and tests. Same deployment-profile separation as API keys — see [CONFIGURATION.md § Deployment profiles](./CONFIGURATION.md#deployment-profiles). + ## Docker image The multi-stage [`Dockerfile`](../Dockerfile): diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 9797f90..8f7195e 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -8,8 +8,8 @@ Success payloads separate **stable** fields (safe across minor bumps after `1.0. | Tool | Stable | Experimental | | ---- | ------ | ------------ | -| `list_sources` | `status`, `sources`, `default` | _(none)_ | -| `list_namespaces` | `status`, `cache_hit`, `cache_ttl_seconds`, `expires_at_iso`, `count`, `namespaces`, optional `source_errors` | _(none)_ | +| `list_sources` | `status`, `sources` (`{ name, description? }[]`), `default` | _(none)_ | +| `list_namespaces` | `status`, `cache_hit`, `cache_ttl_seconds`, `expires_at_iso`, `count`, `namespaces`, optional `source_errors`, optional `config_warnings`, optional per-row `schema_source` | _(none)_ | | `namespace_router` | `status`, `cache_hit`, `user_query`, `suggestions`, `recommended_namespace`, optional `recommended_source` | _(none)_ | | `suggest_query_params` | `status`, `cache_hit`, `suggested_fields`, `recommended_tool`, `use_count_tool`, `explanation`, `namespace_found`, optional `source` | _(none)_ | | `count` | `status`, `count`, `truncated`, `namespace`, `metadata_filter`, optional `source` | _(none)_ | @@ -73,7 +73,7 @@ Registered only when more than one Pinecone source is configured. | | | | --- | --- | | **Input** | _(empty object)_ | -| **Success** | `{ status: 'success', sources: string[], default: string }` | +| **Success** | `{ status: 'success', sources: { name, description? }[], default: string }` | | **Errors** | `LIFECYCLE` when not in multi-source mode | **Example:** @@ -91,7 +91,7 @@ Registered only when more than one Pinecone source is configured. | | | | --- | --- | | **Input** | Optional `source` — filter to one configured project | -| **Success** | `{ status: 'success', cache_hit, cache_ttl_seconds, expires_at_iso, count, namespaces: [{ name, record_count, metadata_fields, source? }], source_errors? }` | +| **Success** | `{ status: 'success', cache_hit, cache_ttl_seconds, expires_at_iso, count, namespaces: [{ name, record_count, metadata_fields, source?, schema_source?, description? }], source_errors?, config_warnings? }` | | **Errors** | `PINECONE_ERROR`, `TIMEOUT`, etc. | **Example (multi-source, all projects):** diff --git a/examples/alliance/demo-mock-pinecone-client.ts b/examples/alliance/demo-mock-pinecone-client.ts index b2b0939..dc56cef 100644 --- a/examples/alliance/demo-mock-pinecone-client.ts +++ b/examples/alliance/demo-mock-pinecone-client.ts @@ -10,6 +10,7 @@ import { type HybridQueryResult, type KeywordIndexNamespacesResult, type KeywordSearchParams, + type NamespacesWithMetadataResult, type PineconeMetadataValue, type QueryParams, type SearchResult, @@ -36,21 +37,26 @@ export class DemoMockPineconeClient extends PineconeClient { super({ apiKey: '00000000-0000-0000-0000-000000000000', indexName: 'demo-index' }); } - override async listNamespacesWithMetadata(): Promise< - Array<{ namespace: string; recordCount: number; metadata: Record }> - > { - return [ - { - namespace: DEMO_NAMESPACE, - recordCount: 42, - metadata: { - document_number: 'string', - title: 'string', - chunk_text: 'string', - url: 'string', + override async listNamespacesWithMetadata( + _declaredSchemas?: Record>, + _declaredNamespaceNames?: string[] + ): Promise { + return { + namespaces: [ + { + namespace: DEMO_NAMESPACE, + recordCount: 42, + metadata: { + document_number: 'string', + title: 'string', + chunk_text: 'string', + url: 'string', + }, + schema_source: 'sampled', }, - }, - ]; + ], + warnings: [], + }; } override async listNamespacesFromKeywordIndex(): Promise { diff --git a/examples/multi-source/pinecone-sources.json.example b/examples/multi-source/pinecone-sources.json.example index 095b9cc..bef18dd 100644 --- a/examples/multi-source/pinecone-sources.json.example +++ b/examples/multi-source/pinecone-sources.json.example @@ -3,7 +3,17 @@ "sources": { "api_key_1": { "apiKey": "${PINECONE_API_KEY_1}", - "indexName": "index_name_1" + "indexName": "index_name_1", + "description": "", + "namespaces": { + "example-namespace": { + "description": "", + "metadata_schema": { + "field_a": "string", + "field_b": "number" + } + } + } }, "api_key_2": { "apiKey": "${PINECONE_API_KEY_2}", diff --git a/scripts/test-search.ts b/scripts/test-search.ts index d63d7a3..2b401d9 100644 --- a/scripts/test-search.ts +++ b/scripts/test-search.ts @@ -36,22 +36,27 @@ async function test() { // Test 1: List namespaces with metadata console.log('\n📋 Test 1: Listing namespaces with metadata...'); const namespacesInfo = await client.listNamespacesWithMetadata(); - console.log(`✅ Found ${namespacesInfo.length} namespace(s):`); - namespacesInfo.forEach((ns) => { + console.log(`✅ Found ${namespacesInfo.namespaces.length} namespace(s):`); + namespacesInfo.namespaces.forEach((ns) => { console.log(` - ${ns.namespace}: ${ns.recordCount} records`); if (Object.keys(ns.metadata).length > 0) { console.log(` Metadata fields:`, Object.keys(ns.metadata)); } }); - if (namespacesInfo.length === 0) { + if (namespacesInfo.warnings.length > 0) { + console.log('⚠️ Config warnings:'); + namespacesInfo.warnings.forEach((warning) => console.log(` - ${warning}`)); + } + + if (namespacesInfo.namespaces.length === 0) { console.log('⚠️ No namespaces found in your index'); console.log(' Make sure your Pinecone index has data'); return; } // Test 2: Query without reranking (faster) - const testNamespace = namespacesInfo[0].namespace; + const testNamespace = namespacesInfo.namespaces[0].namespace; console.log(`\n🔍 Test 2: Query WITHOUT reranking (faster)`); console.log(` Namespace: "${testNamespace}"`); console.log(` Query: "test query"`); @@ -95,7 +100,7 @@ async function test() { } // Test 4: Query with metadata filter on wg21-papers namespace - const wg21Namespace = namespacesInfo.find((ns) => ns.namespace === 'wg21-papers'); + const wg21Namespace = namespacesInfo.namespaces.find((ns) => ns.namespace === 'wg21-papers'); let duration3: number | undefined; if (wg21Namespace) { @@ -149,7 +154,7 @@ async function test() { } else { console.log(`\n⚠️ Test 4 skipped: "wg21-papers" namespace not found`); console.log( - ` Available namespaces: ${namespacesInfo.map((ns) => ns.namespace).join(', ')}` + ` Available namespaces: ${namespacesInfo.namespaces.map((ns) => ns.namespace).join(', ')}` ); } diff --git a/src/alliance/tools/isolated-context.context.test.ts b/src/alliance/tools/isolated-context.context.test.ts index 6c46137..c0a49cf 100644 --- a/src/alliance/tools/isolated-context.context.test.ts +++ b/src/alliance/tools/isolated-context.context.test.ts @@ -7,6 +7,7 @@ import { isolateFromDefaultContext, makeHybridQueryResult, makeSearchResult, + mockNamespacesWithMetadataResult, parseToolJson, } from '../../core/server/tools/test-helpers.js'; import { @@ -27,18 +28,20 @@ describe('isolated ServerContext with zero default context', () => { }); it('guided_query enrich_urls uses ctx builtins, not default registry', async () => { - const listNamespacesWithMetadata = vi.fn().mockResolvedValue([ - { - namespace: 'mailing', - recordCount: 42, - metadata: { - document_number: 'string', - title: 'string', - author: 'string', - chunk_text: 'string', + const listNamespacesWithMetadata = vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'mailing', + recordCount: 42, + metadata: { + document_number: 'string', + title: 'string', + author: 'string', + chunk_text: 'string', + }, }, - }, - ]); + ]) + ); const query = vi.fn().mockResolvedValue( makeHybridQueryResult({ results: [ diff --git a/src/alliance/tools/suggest-query-params-tool.context.test.ts b/src/alliance/tools/suggest-query-params-tool.context.test.ts index ccb81c0..829c78a 100644 --- a/src/alliance/tools/suggest-query-params-tool.context.test.ts +++ b/src/alliance/tools/suggest-query-params-tool.context.test.ts @@ -7,6 +7,7 @@ import { createTestServerContext, expectMatchesResponseSchema, makeHybridQueryResult, + mockNamespacesWithMetadataResult, parseToolJson, } from '../../core/server/tools/test-helpers.js'; @@ -22,7 +23,11 @@ function mockNamespacesClient() { return { listNamespacesWithMetadata: vi .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 42, metadata: namespaceMetadata }]), + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'wg21', recordCount: 42, metadata: namespaceMetadata }, + ]) + ), }; } @@ -32,19 +37,21 @@ describe('suggest_query_params tool handler (ServerContext instance path)', () = }); it('marks suggest-flow on injected context when namespace exists', async () => { - const listNamespacesWithMetadata = vi.fn().mockResolvedValue([ - { - namespace: 'wg21', - recordCount: 42, - metadata: { - document_number: 'string', - title: 'string', - url: 'string', - author: 'string', - chunk_text: 'string', + const listNamespacesWithMetadata = vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'wg21', + recordCount: 42, + metadata: { + document_number: 'string', + title: 'string', + url: 'string', + author: 'string', + chunk_text: 'string', + }, }, - }, - ]); + ]) + ); const ctx = createTestServerContext({ client: { listNamespacesWithMetadata } as never, }); diff --git a/src/constants.test.ts b/src/constants.test.ts index 9b613bd..70bd2c1 100644 --- a/src/constants.test.ts +++ b/src/constants.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { describe, expect, it } from 'vitest'; import { ALLIANCE_INSTRUCTIONS_APPENDIX, @@ -55,4 +57,35 @@ describe('server instructions', () => { it('SERVER_INSTRUCTIONS aliases Alliance instructions', () => { expect(SERVER_INSTRUCTIONS).toBe(ALLIANCE_SERVER_INSTRUCTIONS); }); + + it('example multi-source config uses only generic placeholder descriptions and schemas', () => { + const examplePath = join(process.cwd(), 'examples/multi-source/pinecone-sources.json.example'); + const raw = readFileSync(examplePath, 'utf8'); + const parsed = JSON.parse(raw) as { + sources: Record< + string, + { + description?: string; + namespaces?: Record< + string, + { description?: string; metadata_schema?: Record } + >; + } + >; + }; + const corpusPlaceholder = + ''; + const namespacePlaceholder = + ''; + expect(parsed.sources.api_key_1.description).toBe(corpusPlaceholder); + expect(parsed.sources.api_key_1.namespaces?.['example-namespace']?.description).toBe( + namespacePlaceholder + ); + expect(parsed.sources.api_key_1.namespaces?.['example-namespace']?.metadata_schema).toEqual({ + field_a: 'string', + field_b: 'number', + }); + expect(CORE_SERVER_INSTRUCTIONS).not.toContain(corpusPlaceholder); + expect(ALLIANCE_SERVER_INSTRUCTIONS).not.toContain(corpusPlaceholder); + }); }); diff --git a/src/core/index.ts b/src/core/index.ts index f603331..1578a67 100644 --- a/src/core/index.ts +++ b/src/core/index.ts @@ -79,6 +79,7 @@ export type { ConfigOverrides, } from './config.js'; export { PineconeClient } from './pinecone-client.js'; +export type { NamespacesWithMetadataResult, NamespaceWithMetadataRow } from './pinecone/indexes.js'; export type { PineconeClientConfig, QueryParams, diff --git a/src/core/pinecone-client.ts b/src/core/pinecone-client.ts index 7ba81d3..1f7cd89 100644 --- a/src/core/pinecone-client.ts +++ b/src/core/pinecone-client.ts @@ -15,7 +15,7 @@ import type { HybridLegFailed, } from '../types.js'; import { DEFAULT_TOP_K, MAX_TOP_K, COUNT_TOP_K, COUNT_FIELDS } from '../constants.js'; -import { PineconeIndexSession } from './pinecone/indexes.js'; +import { PineconeIndexSession, type NamespacesWithMetadataResult } from './pinecone/indexes.js'; import { countUniqueDocumentsFromHits, mapSparseHitsToSearchResults, @@ -77,15 +77,12 @@ export class PineconeClient { return this.indexSession.listNamespacesFromKeywordIndex(); } - /** Dense index namespaces with sampled metadata field types. */ - async listNamespacesWithMetadata(): Promise< - Array<{ - namespace: string; - recordCount: number; - metadata: Record; - }> - > { - return this.indexSession.listNamespacesWithMetadata(); + /** Dense index namespaces with sampled or declared metadata field types. */ + async listNamespacesWithMetadata( + declaredSchemas?: Record>, + declaredNamespaceNames?: string[] + ): Promise { + return this.indexSession.listNamespacesWithMetadata(declaredSchemas, declaredNamespaceNames); } /** Probe dense + sparse indexes (describeIndexStats) for startup checks. */ diff --git a/src/core/pinecone/indexes.test.ts b/src/core/pinecone/indexes.test.ts index 47520fa..ac7deb2 100644 --- a/src/core/pinecone/indexes.test.ts +++ b/src/core/pinecone/indexes.test.ts @@ -79,7 +79,8 @@ describe('PineconeIndexSession', () => { }); const rows = await session.listNamespacesWithMetadata(); - expect(rows).toEqual([]); + expect(rows.namespaces).toEqual([]); + expect(rows.warnings).toEqual([]); }); it('returns row with empty metadata when recordCount is zero', async () => { @@ -95,8 +96,13 @@ describe('PineconeIndexSession', () => { }); const rows = await session.listNamespacesWithMetadata(); - expect(rows).toHaveLength(1); - expect(rows[0]).toEqual({ namespace: 'ns1', recordCount: 0, metadata: {} }); + expect(rows.namespaces).toHaveLength(1); + expect(rows.namespaces[0]).toEqual({ + namespace: 'ns1', + recordCount: 0, + metadata: {}, + schema_source: 'sampled', + }); }); it('samples metadata when records exist and namespace.query returns matches', async () => { @@ -126,12 +132,105 @@ describe('PineconeIndexSession', () => { }); const rows = await session.listNamespacesWithMetadata(); - expect(rows).toHaveLength(1); - expect(rows[0]?.namespace).toBe('ns1'); - expect(rows[0]?.metadata['title']).toBe('string'); - expect(rows[0]?.metadata['tags']).toBe('string[]'); - expect(rows[0]?.metadata['emptyArr']).toBe('array'); - expect(rows[0]?.metadata['nested']).toBe('object'); + expect(rows.namespaces).toHaveLength(1); + expect(rows.namespaces[0]?.namespace).toBe('ns1'); + expect(rows.namespaces[0]?.metadata['title']).toBe('string'); + expect(rows.namespaces[0]?.metadata['tags']).toBe('string[]'); + expect(rows.namespaces[0]?.metadata['emptyArr']).toBe('array'); + expect(rows.namespaces[0]?.metadata['nested']).toBe('object'); + expect(rows.namespaces[0]?.schema_source).toBe('sampled'); + }); + + it('uses declared schema and skips sampling when declaredSchemas provides one', async () => { + const query = vi.fn(); + const dense = { + describeIndexStats: vi.fn().mockResolvedValue({ + namespaces: { declared_ns: { recordCount: 5 }, sampled_ns: { recordCount: 2 } }, + dimension: 4, + }), + namespace: () => ({ query }), + } as unknown as SearchableIndex; + const session = new PineconeIndexSessionTestDouble({ + dense, + sparse: {} as SearchableIndex, + }); + + const rows = await session.listNamespacesWithMetadata({ + declared_ns: { title: 'string', author: 'string' }, + }); + + const declared = rows.namespaces.find((n) => n.namespace === 'declared_ns'); + expect(declared).toMatchObject({ + metadata: { title: 'string', author: 'string' }, + schema_source: 'declared', + }); + const sampled = rows.namespaces.find((n) => n.namespace === 'sampled_ns'); + expect(sampled?.schema_source).toBe('sampled'); + expect(query).toHaveBeenCalledOnce(); + }); + + it('warns when declared namespace is missing from live Pinecone index', async () => { + const dense = { + describeIndexStats: vi.fn().mockResolvedValue({ + namespaces: { live_ns: { recordCount: 1 } }, + }), + namespace: vi.fn(), + } as unknown as SearchableIndex; + const session = new PineconeIndexSessionTestDouble({ + dense, + sparse: {} as SearchableIndex, + }); + + const rows = await session.listNamespacesWithMetadata({ + stale_ns: { title: 'string' }, + }); + + expect(rows.namespaces.map((n) => n.namespace)).toEqual(['live_ns']); + expect(rows.warnings.some((w) => w.includes('stale_ns'))).toBe(true); + }); + + it('warns when declaredNamespaceNames includes description-only stale namespace', async () => { + const dense = { + describeIndexStats: vi.fn().mockResolvedValue({ + namespaces: { live_ns: { recordCount: 1 } }, + }), + namespace: vi.fn(), + } as unknown as SearchableIndex; + const session = new PineconeIndexSessionTestDouble({ + dense, + sparse: {} as SearchableIndex, + }); + + const rows = await session.listNamespacesWithMetadata(undefined, ['desc_only_stale']); + + expect(rows.namespaces.map((n) => n.namespace)).toEqual(['live_ns']); + expect(rows.warnings.some((w) => w.includes('desc_only_stale'))).toBe(true); + }); + + it('samples description-only live namespace when no metadata_schema declared', async () => { + const query = vi.fn().mockResolvedValue({ + matches: [{ metadata: { title: 'T' } }], + }); + const dense = { + describeIndexStats: vi.fn().mockResolvedValue({ + namespaces: { live_ns: { recordCount: 2 } }, + dimension: 4, + }), + namespace: () => ({ query }), + } as unknown as SearchableIndex; + const session = new PineconeIndexSessionTestDouble({ + dense, + sparse: {} as SearchableIndex, + }); + + const rows = await session.listNamespacesWithMetadata(undefined, ['live_ns']); + + expect(rows.warnings).toEqual([]); + expect(rows.namespaces[0]).toMatchObject({ + namespace: 'live_ns', + schema_source: 'sampled', + }); + expect(query).toHaveBeenCalledOnce(); }); }); diff --git a/src/core/pinecone/indexes.ts b/src/core/pinecone/indexes.ts index 63099ed..c519c63 100644 --- a/src/core/pinecone/indexes.ts +++ b/src/core/pinecone/indexes.ts @@ -10,10 +10,6 @@ import type { SearchableIndex, } from '../../types.js'; -/** - * Infers a human-readable metadata field type for namespace discovery. - * Distinguishes Pinecone-supported list type (string[]) from other arrays. - */ function inferMetadataFieldType(value: unknown): string { if (value === null || value === undefined) { return 'unknown'; @@ -28,6 +24,18 @@ function inferMetadataFieldType(value: unknown): string { return 'object'; } +export type NamespaceWithMetadataRow = { + namespace: string; + recordCount: number; + metadata: Record; + schema_source: 'declared' | 'sampled'; +}; + +export type NamespacesWithMetadataResult = { + namespaces: NamespaceWithMetadataRow[]; + warnings: string[]; +}; + /** Holds lazy Pinecone SDK client and dense/sparse index references. */ export class PineconeIndexSession { private pc: Pinecone | null = null; @@ -112,15 +120,13 @@ export class PineconeIndexSession { * List all available namespaces with their metadata information * * Fetches namespaces from the index stats and samples records to discover - * available metadata fields and their types. + * available metadata fields and their types. When `declaredSchemas` provides + * a schema for a live namespace, sampling is skipped for that namespace. */ - async listNamespacesWithMetadata(): Promise< - Array<{ - namespace: string; - recordCount: number; - metadata: Record; - }> - > { + async listNamespacesWithMetadata( + declaredSchemas?: Record>, + declaredNamespaceNames?: string[] + ): Promise { try { const { denseIndex } = await this.ensureIndexes(); @@ -129,14 +135,36 @@ export class PineconeIndexSession { ? await denseIndex.describeIndexStats() : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; + const liveSet = new Set(namespaces); logInfo(`Found ${namespaces.length} namespace(s)`); - // Get metadata info for each namespace by sampling records + const warnings: string[] = []; + const namesToVerify = + declaredNamespaceNames ?? (declaredSchemas ? Object.keys(declaredSchemas) : []); + for (const declaredNs of namesToVerify) { + if (!liveSet.has(declaredNs)) { + warnings.push( + `Declared namespace "${declaredNs}" not found in Pinecone index "${this.indexName}" — schema declaration is stale.` + ); + } + } + + // Get metadata info for each namespace by sampling records (or use declared schema) const namespacesInfo = await Promise.all( namespaces.map(async (ns: string) => { try { const recordCount = stats?.namespaces?.[ns]?.recordCount || 0; + const declared = declaredSchemas?.[ns]; + if (declared) { + return { + namespace: ns, + recordCount, + metadata: { ...declared }, + schema_source: 'declared' as const, + }; + } + const metadataFields: Record = {}; // Sample a few records to discover metadata fields @@ -180,6 +208,7 @@ export class PineconeIndexSession { namespace: ns, recordCount, metadata: metadataFields, + schema_source: 'sampled' as const, }; } catch (error) { logError(`Error processing namespace ${ns}`, error); @@ -187,15 +216,16 @@ export class PineconeIndexSession { namespace: ns, recordCount: 0, metadata: {}, + schema_source: 'sampled' as const, }; } }) ); - return namespacesInfo; + return { namespaces: namespacesInfo, warnings }; } catch (error) { logError('Error listing namespaces', error); - return []; + return { namespaces: [], warnings: [] }; } } diff --git a/src/core/server/namespace-cache.ts b/src/core/server/namespace-cache.ts new file mode 100644 index 0000000..d42714c --- /dev/null +++ b/src/core/server/namespace-cache.ts @@ -0,0 +1,45 @@ +import type { PineconeClient } from '../pinecone-client.js'; +import type { NamespaceWithMetadataRow } from '../pinecone/indexes.js'; +import { extractDeclaredSchemas, type NamespaceDeclaration } from './source-config.js'; +import type { NamespaceInfo } from './server-context.js'; + +export type NamespacesCacheEntry = { + data: NamespaceInfo[]; + expiresAt: number; + warnings: string[]; +}; + +export function mapNamespaceRowsToInfo( + rows: NamespaceWithMetadataRow[], + options?: { + declaredNamespaces?: Record; + source?: string; + } +): NamespaceInfo[] { + const { declaredNamespaces, source } = options ?? {}; + return rows.map((ns) => { + const description = declaredNamespaces?.[ns.namespace]?.description; + return { + namespace: ns.namespace, + recordCount: ns.recordCount, + metadata: ns.metadata, + schema_source: ns.schema_source, + ...(source !== undefined ? { source } : {}), + ...(description !== undefined ? { description } : {}), + }; + }); +} + +export async function fetchNamespacesWithDeclaredConfig( + client: PineconeClient, + declaredNamespaces?: Record, + source?: string +): Promise<{ data: NamespaceInfo[]; warnings: string[] }> { + const declaredSchemas = extractDeclaredSchemas(declaredNamespaces); + const declaredNamespaceNames = declaredNamespaces ? Object.keys(declaredNamespaces) : undefined; + const raw = await client.listNamespacesWithMetadata(declaredSchemas, declaredNamespaceNames); + return { + data: mapNamespaceRowsToInfo(raw.namespaces, { declaredNamespaces, source }), + warnings: raw.warnings, + }; +} diff --git a/src/core/server/namespaces-cache.ts b/src/core/server/namespaces-cache.ts index 5fdba93..3d8d27a 100644 --- a/src/core/server/namespaces-cache.ts +++ b/src/core/server/namespaces-cache.ts @@ -16,6 +16,8 @@ export async function getNamespacesWithCache(): Promise<{ data: NamespaceInfo[]; cache_hit: boolean; expires_at: number; + source_errors?: Record; + warnings?: string[]; }> { warnLegacyFacade('getNamespacesWithCache'); return resolveDefaultServerContext().getNamespacesWithCache(); diff --git a/src/core/server/response-schemas.test.ts b/src/core/server/response-schemas.test.ts index 22301f8..3bd085f 100644 --- a/src/core/server/response-schemas.test.ts +++ b/src/core/server/response-schemas.test.ts @@ -6,6 +6,7 @@ import { keywordSearchResponseSchema, keywordSearchSuccessResponseSchema, listNamespacesResponseSchema, + listSourcesResponseSchema, namespaceRouterResponseSchema, queryDocumentsResponseSchema, queryResponseSchema, @@ -195,6 +196,33 @@ describe('response-schemas', () => { }); it('validates all nine tool schemas with minimal fixtures', () => { + expect( + listNamespacesResponseSchema.parse({ + status: 'success', + cache_hit: false, + cache_ttl_seconds: 1800, + expires_at_iso: '2026-01-01T00:00:00.000Z', + count: 1, + namespaces: [ + { + name: 'wg21', + record_count: 1, + metadata_fields: { title: 'string' }, + schema_source: 'declared', + }, + ], + config_warnings: ['Declared namespace "stale" not found'], + }) + ).toBeDefined(); + + expect( + listSourcesResponseSchema.parse({ + status: 'success', + sources: [{ name: 'api_key_1', description: 'Public corpus' }, { name: 'api_key_2' }], + default: 'api_key_1', + }) + ).toBeDefined(); + expect( listNamespacesResponseSchema.parse({ status: 'success', diff --git a/src/core/server/response-schemas.ts b/src/core/server/response-schemas.ts index d428a9b..da95fa4 100644 --- a/src/core/server/response-schemas.ts +++ b/src/core/server/response-schemas.ts @@ -111,12 +111,15 @@ export const listNamespacesResponseSchema = z.object({ expires_at_iso: z.string(), count: z.number(), source_errors: z.record(z.string(), z.string()).optional(), + config_warnings: z.array(z.string()).optional(), namespaces: z.array( z.object({ name: z.string(), source: z.string().optional(), record_count: z.number(), metadata_fields: z.record(z.string(), z.string()), + schema_source: z.enum(['declared', 'sampled']).optional(), + description: z.string().optional(), }) ), }); @@ -254,7 +257,12 @@ export type GenerateUrlsResponse = z.infer; export const listSourcesResponseSchema = z.object({ status: z.literal('success'), - sources: z.array(z.string()), + sources: z.array( + z.object({ + name: z.string(), + description: z.string().optional(), + }) + ), default: z.string(), }); diff --git a/src/core/server/server-context.composition.test.ts b/src/core/server/server-context.composition.test.ts index 7dd424d..4de0258 100644 --- a/src/core/server/server-context.composition.test.ts +++ b/src/core/server/server-context.composition.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { PineconeClient } from '../pinecone-client.js'; -import { resolveTestConfig } from './tools/test-helpers.js'; +import { resolveTestConfig, mockNamespacesWithMetadataResult } from './tools/test-helpers.js'; import { ServerContext, createIsolatedContext, @@ -88,7 +88,11 @@ describe('ServerContext composition API', () => { const config = testConfig(); const listNamespaces = vi .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 1, metadata: { title: 'string' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'wg21', recordCount: 1, metadata: { title: 'string' } }, + ]) + ); const injected = { listNamespacesWithMetadata: listNamespaces } as never; const viaComposition = new ServerContext(config, { client: injected }); @@ -106,7 +110,11 @@ describe('ServerContext composition API', () => { it('refetches when namespace cache seed is expired', async () => { const listNamespaces = vi .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 2, metadata: { title: 'string' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'wg21', recordCount: 2, metadata: { title: 'string' } }, + ]) + ); const ctx = new ServerContext(testConfig(), { client: { listNamespacesWithMetadata: listNamespaces } as never, namespaceCacheSeed: { @@ -124,7 +132,11 @@ describe('ServerContext composition API', () => { it('setConfig preserves URL generators but clears namespace cache and suggest-flow', async () => { const listNamespaces = vi .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 1, metadata: { title: 'string' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'wg21', recordCount: 1, metadata: { title: 'string' } }, + ]) + ); const ctx = new ServerContext(testConfig(), { client: { listNamespacesWithMetadata: listNamespaces } as never, urlGenerators: [['wg21', () => ({ url: 'https://example.com', method: 'generated.custom' })]], @@ -207,10 +219,18 @@ describe('ServerContext composition API', () => { it('isolates namespace cache between two createIsolatedContext instances', async () => { const listA = vi .fn() - .mockResolvedValue([{ namespace: 'a', recordCount: 1, metadata: { source: 'a' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'a', recordCount: 1, metadata: { source: 'a' } }, + ]) + ); const listB = vi .fn() - .mockResolvedValue([{ namespace: 'b', recordCount: 2, metadata: { source: 'b' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'b', recordCount: 2, metadata: { source: 'b' } }, + ]) + ); const cfgA = resolveTestConfig({ apiKey: 'iso-a' }); const cfgB = resolveTestConfig({ apiKey: 'iso-b' }); const seed = { diff --git a/src/core/server/server-context.test.ts b/src/core/server/server-context.test.ts index 8f85949..69ba0e4 100644 --- a/src/core/server/server-context.test.ts +++ b/src/core/server/server-context.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { PineconeClient } from '../pinecone-client.js'; -import { resolveTestConfig } from './tools/test-helpers.js'; +import { resolveTestConfig, mockNamespacesWithMetadataResult } from './tools/test-helpers.js'; import { ServerContext, createServer, @@ -50,10 +50,18 @@ describe('ServerContext', () => { it('setConfig clears client, namespaces cache, and suggest-flow', async () => { const listA = vi .fn() - .mockResolvedValue([{ namespace: 'a', recordCount: 1, metadata: { title: 'string' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'a', recordCount: 1, metadata: { title: 'string' } }, + ]) + ); const listB = vi .fn() - .mockResolvedValue([{ namespace: 'b', recordCount: 2, metadata: { title: 'string' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'b', recordCount: 2, metadata: { title: 'string' } }, + ]) + ); const ctx = ServerContext.fromClient(testConfig(), { listNamespacesWithMetadata: listA, } as never); @@ -111,7 +119,11 @@ describe('ServerContext', () => { const config = testConfig(); const listNamespaces = vi .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 1, metadata: { title: 'string' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'wg21', recordCount: 1, metadata: { title: 'string' } }, + ]) + ); const ctx = ServerContext.fromClient(config, { listNamespacesWithMetadata: listNamespaces, } as never); diff --git a/src/core/server/server-context.ts b/src/core/server/server-context.ts index 7ef2dcb..9ef3f21 100644 --- a/src/core/server/server-context.ts +++ b/src/core/server/server-context.ts @@ -11,12 +11,15 @@ import { normalizeNamespace } from './namespace-utils.js'; import type { RecommendedTool } from './query-suggestion.js'; import type { UrlGenerationResult, UrlGeneratorFn } from './url-registry.js'; import { buildSourceRegistry, type SourceRegistry } from './source-registry.js'; +import { fetchNamespacesWithDeclaredConfig, type NamespacesCacheEntry } from './namespace-cache.js'; export type NamespaceInfo = { namespace: string; recordCount: number; metadata: Record; source?: string; + schema_source?: 'declared' | 'sampled'; + description?: string; }; export type ResolveSourceResult = @@ -70,12 +73,6 @@ type FlowState = { user_query: string; }; -type CacheEntry = { - data: NamespaceInfo[]; - expiresAt: number; -}; - -/** Return a trimmed non-empty string or null for empty/missing values. */ function asString(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; } @@ -125,7 +122,7 @@ export class ServerContext< private readonly unconfiguredAlliance: boolean; private readonly urlGenerators = new Map(); private readonly suggestionFlow = new Map(); - private namespacesCache: CacheEntry | null = null; + private namespacesCache: NamespacesCacheEntry | null = null; constructor(config?: T, composition?: ServerContextComposition, init?: ServerContextInitOptions) { this.unconfiguredAlliance = init?.unconfiguredAlliance ?? false; @@ -156,6 +153,7 @@ export class ServerContext< metadata: { ...entry.metadata }, })), expiresAt, + warnings: [], }; } if (composition?.suggestionFlowSeed) { @@ -259,6 +257,20 @@ export class ServerContext< return []; } + listSourceDetails(): { name: string; description?: string }[] { + if (this.sourceRegistry) { + return this.sourceRegistry.listSources().map((name) => { + const def = this.sourceRegistry!.getDefinition(name); + return { name, ...(def.description !== undefined ? { description: def.description } : {}) }; + }); + } + const cfg = this.getConfigIfSet() ?? this.getConfig(); + return (cfg.sources ?? []).map((s) => ({ + name: s.name, + ...(s.description !== undefined ? { description: s.description } : {}), + })); + } + getDefaultSourceName(): string { if (this.sourceRegistry) { return this.sourceRegistry.getDefaultName(); @@ -582,6 +594,7 @@ export class ServerContext< cache_hit: boolean; expires_at: number; source_errors?: Record; + warnings?: string[]; }> { if ( this.isMultiSource() || @@ -602,15 +615,22 @@ export class ServerContext< data: this.namespacesCache.data, cache_hit: true, expires_at: this.namespacesCache.expiresAt, + ...(this.namespacesCache.warnings.length > 0 + ? { warnings: [...this.namespacesCache.warnings] } + : {}), }; } - const client = this.getClient(); - const data = await client.listNamespacesWithMetadata(); - const ttlMs = this.getConfig().cacheTtlMs; - const expiresAt = now + ttlMs; - this.namespacesCache = { data, expiresAt }; - return { data, cache_hit: false, expires_at: expiresAt }; + const cfg = this.getConfig(); + const { data, warnings } = await fetchNamespacesWithDeclaredConfig(this.getClient()); + const expiresAt = now + cfg.cacheTtlMs; + this.namespacesCache = { data, expiresAt, warnings }; + return { + data, + cache_hit: false, + expires_at: expiresAt, + ...(warnings.length > 0 ? { warnings: [...warnings] } : {}), + }; } async getNamespacesWithCacheForSource(source: string): Promise<{ diff --git a/src/core/server/source-config.test.ts b/src/core/server/source-config.test.ts index fe79cbc..fee1f92 100644 --- a/src/core/server/source-config.test.ts +++ b/src/core/server/source-config.test.ts @@ -104,6 +104,118 @@ describe('source-config', () => { expect(() => parseSourcesConfigFile(filePath, {})).toThrow(/defaultSource/); }); + it('parseSourcesConfigFile reads description and namespaces with metadata_schema', () => { + const dir = mkdtempSync(join(tmpdir(), 'pinecone-sources-')); + const filePath = join(dir, 'sources.json'); + writeFileSync( + filePath, + JSON.stringify({ + defaultSource: 'api_key_1', + sources: { + api_key_1: { + apiKey: 'k', + indexName: 'index_name_1', + description: 'Staff corpus hint', + namespaces: { + example_ns: { + description: 'Namespace hint', + metadata_schema: { field_a: 'string', field_b: 'number' }, + }, + }, + }, + }, + }) + ); + const parsed = parseSourcesConfigFile(filePath, {}); + const first = parsed.sources[0]; + expect(first?.description).toBe('Staff corpus hint'); + expect(first?.namespaces?.example_ns).toMatchObject({ + description: 'Namespace hint', + metadata_schema: { field_a: 'string', field_b: 'number' }, + }); + }); + + it('parseSourcesConfigFile omits description and namespaces when not set (back-compat)', () => { + const dir = mkdtempSync(join(tmpdir(), 'pinecone-sources-')); + const filePath = join(dir, 'minimal.json'); + writeFileSync( + filePath, + JSON.stringify({ + defaultSource: 'api_key_1', + sources: { + api_key_1: { apiKey: 'k', indexName: 'idx' }, + }, + }) + ); + const parsed = parseSourcesConfigFile(filePath, {}); + const first = parsed.sources[0]; + expect(first?.description).toBeUndefined(); + expect(first?.namespaces).toBeUndefined(); + }); + + it('parseSourcesConfigFile throws when source description is not a string', () => { + const dir = mkdtempSync(join(tmpdir(), 'pinecone-sources-')); + const filePath = join(dir, 'bad-source-description.json'); + writeFileSync( + filePath, + JSON.stringify({ + defaultSource: 'api_key_1', + sources: { + api_key_1: { apiKey: 'k', indexName: 'idx', description: 42 }, + }, + }) + ); + expect(() => parseSourcesConfigFile(filePath, {})).toThrow( + /Source "api_key_1": description must be a string/ + ); + }); + + it('parseSourcesConfigFile throws when namespace description is not a string', () => { + const dir = mkdtempSync(join(tmpdir(), 'pinecone-sources-')); + const filePath = join(dir, 'bad-namespace-description.json'); + writeFileSync( + filePath, + JSON.stringify({ + defaultSource: 'api_key_1', + sources: { + api_key_1: { + apiKey: 'k', + indexName: 'idx', + namespaces: { example_ns: { description: 42 } }, + }, + }, + }) + ); + expect(() => parseSourcesConfigFile(filePath, {})).toThrow( + /namespaces\["example_ns"\]\.description must be a string/ + ); + }); + + it('parseSourcesConfigFile throws on malformed metadata_schema value', () => { + const dir = mkdtempSync(join(tmpdir(), 'pinecone-sources-')); + const filePath = join(dir, 'bad-schema.json'); + writeFileSync( + filePath, + JSON.stringify({ + defaultSource: 'api_key_1', + sources: { + api_key_1: { + apiKey: 'k', + indexName: 'idx', + namespaces: { ns1: { metadata_schema: { field_a: 123 } } }, + }, + }, + }) + ); + expect(() => parseSourcesConfigFile(filePath, {})).toThrow(/metadata_schema/); + }); + + it('parseInlineSources never includes description or namespaces', () => { + const sources = parseInlineSources('api_key_1:sk:idx', {}); + expect(sources[0]).not.toHaveProperty('description'); + expect(sources[0]).not.toHaveProperty('namespaces'); + }); + it('resolveConfig uses PINECONE_SOURCES when set', () => { vi.stubEnv('PINECONE_SOURCES', 'api_key_1:sk-test:my-index'); vi.stubEnv('PINECONE_API_KEY', 'ignored'); diff --git a/src/core/server/source-config.ts b/src/core/server/source-config.ts index 5a64c1a..ad04f49 100644 --- a/src/core/server/source-config.ts +++ b/src/core/server/source-config.ts @@ -6,6 +6,12 @@ import { readFileSync } from 'node:fs'; import { resolve } from 'node:path'; import { trimOptional } from '../config.js'; +/** Per-namespace declaration loaded from private JSON config (config-file only). */ +export interface NamespaceDeclaration { + description?: string; + metadata_schema?: Record; +} + /** Named Pinecone project connection (one API key + index pair). */ export interface SourceDefinition { name: string; @@ -13,6 +19,10 @@ export interface SourceDefinition { indexName: string; sparseIndexName?: string; rerankModel?: string; + /** Optional corpus-level description (config-file only; never from inline PINECONE_SOURCES). */ + description?: string; + /** Optional per-namespace declarations (config-file only). */ + namespaces?: Record; } export type ParseSourcesOptions = { @@ -48,6 +58,60 @@ function validateSourceName(name: string): void { } } +function validateNamespaces( + sourceName: string, + raw: unknown +): Record | undefined { + if (raw === undefined || raw === null) { + return undefined; + } + if (typeof raw !== 'object' || Array.isArray(raw)) { + throw new Error(`Source "${sourceName}": namespaces must be an object.`); + } + const result: Record = {}; + for (const [nsName, entry] of Object.entries(raw as Record)) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new Error(`Source "${sourceName}": namespaces["${nsName}"] must be an object.`); + } + const obj = entry as Record; + const declaration: NamespaceDeclaration = {}; + if (obj['description'] != null) { + if (typeof obj['description'] !== 'string') { + throw new Error( + `Source "${sourceName}": namespaces["${nsName}"].description must be a string.` + ); + } + const trimmed = obj['description'].trim(); + if (trimmed) { + declaration.description = trimmed; + } + } + if (obj['metadata_schema'] != null) { + if (typeof obj['metadata_schema'] !== 'object' || Array.isArray(obj['metadata_schema'])) { + throw new Error( + `Source "${sourceName}": namespaces["${nsName}"].metadata_schema must be a flat object.` + ); + } + const schema: Record = {}; + for (const [field, type] of Object.entries( + obj['metadata_schema'] as Record + )) { + if (typeof type !== 'string' || !type.trim()) { + throw new Error( + `Source "${sourceName}": namespaces["${nsName}"].metadata_schema["${field}"] must be a non-empty string type.` + ); + } + schema[field] = type.trim(); + } + if (Object.keys(schema).length > 0) { + declaration.metadata_schema = schema; + } + } + result[nsName] = declaration; + } + return Object.keys(result).length > 0 ? result : undefined; +} + function normalizeSourceEntry( name: string, raw: { @@ -55,6 +119,8 @@ function normalizeSourceEntry( indexName: string; sparseIndexName?: string; rerankModel?: string; + description?: string; + namespaces?: Record; }, env: NodeJS.ProcessEnv, allianceDefaults?: ParseSourcesOptions['allianceDefaults'] @@ -82,9 +148,27 @@ function normalizeSourceEntry( indexName, sparseIndexName, ...(rerankModel !== undefined ? { rerankModel } : {}), + ...(raw.description !== undefined ? { description: raw.description } : {}), + ...(raw.namespaces !== undefined ? { namespaces: raw.namespaces } : {}), }; } +/** Extract declared metadata schemas per namespace for Pinecone discovery. */ +export function extractDeclaredSchemas( + namespaces?: Record +): Record> | undefined { + if (!namespaces) { + return undefined; + } + const result: Record> = {}; + for (const [nsName, decl] of Object.entries(namespaces)) { + if (decl.metadata_schema && Object.keys(decl.metadata_schema).length > 0) { + result[nsName] = { ...decl.metadata_schema }; + } + } + return Object.keys(result).length > 0 ? result : undefined; +} + /** Parse inline `name:apiKey:indexName[;name2:...]` format. */ export function parseInlineSources( inline: string, @@ -133,6 +217,8 @@ type JsonSourceFile = { indexName: string; sparseIndexName?: string; rerankModel?: string; + description?: string; + namespaces?: Record; } >; }; @@ -169,6 +255,14 @@ export function parseSourcesConfigFile( if (!cfg || typeof cfg !== 'object') { throw new Error(`Source "${name}" in config file must be an object.`); } + let description: string | undefined; + if (cfg.description != null) { + if (typeof cfg.description !== 'string') { + throw new Error(`Source "${name}": description must be a string.`); + } + description = trimOptional(cfg.description); + } + const namespaces = validateNamespaces(name, cfg.namespaces); sources.push( normalizeSourceEntry( name, @@ -177,6 +271,8 @@ export function parseSourcesConfigFile( indexName: String(cfg.indexName ?? ''), ...(cfg.sparseIndexName != null ? { sparseIndexName: String(cfg.sparseIndexName) } : {}), ...(cfg.rerankModel != null ? { rerankModel: String(cfg.rerankModel) } : {}), + ...(description !== undefined ? { description } : {}), + ...(namespaces !== undefined ? { namespaces } : {}), }, env, options?.allianceDefaults diff --git a/src/core/server/source-registry.test.ts b/src/core/server/source-registry.test.ts index c9ab679..e34aef9 100644 --- a/src/core/server/source-registry.test.ts +++ b/src/core/server/source-registry.test.ts @@ -9,9 +9,17 @@ const sources: SourceDefinition[] = [ function mockClient(name: string) { return { - listNamespacesWithMetadata: vi - .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 10, metadata: { title: 'string' } }]), + listNamespacesWithMetadata: vi.fn().mockResolvedValue({ + namespaces: [ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + schema_source: 'sampled', + }, + ], + warnings: [], + }), checkIndexes: vi.fn().mockResolvedValue({ ok: true, errors: [] }), getSparseIndexName: () => `${name}-sparse`, }; @@ -82,4 +90,91 @@ describe('SourceRegistry', () => { expect(result.source_errors).toEqual({ api_key_2: 'api_key_2 unreachable' }); expect(result.cache_hit).toBe(false); }); + + it('aggregates warnings across sources in getAllNamespacesWithCache', async () => { + const client1 = { + listNamespacesWithMetadata: vi.fn().mockResolvedValue({ + namespaces: [ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + schema_source: 'sampled', + }, + ], + warnings: [ + 'Declared namespace "stale" not found in Pinecone index "idx-a" — schema declaration is stale.', + ], + }), + checkIndexes: vi.fn().mockResolvedValue({ ok: true, errors: [] }), + getSparseIndexName: () => 'idx-a-sparse', + }; + const client2 = mockClient('api_key_2'); + const registry = buildSourceRegistry({ + sources: [ + { + ...sources[0]!, + namespaces: { stale: { metadata_schema: { title: 'string' } } }, + }, + sources[1]!, + ], + defaultSource: 'api_key_1', + cacheTtlMs: 60_000, + defaultTopK: 10, + requestTimeoutMs: 15_000, + clients: new Map([ + ['api_key_1', client1 as never], + ['api_key_2', client2 as never], + ]), + }); + const result = await registry.getAllNamespacesWithCache(); + expect(result.warnings?.some((w) => w.includes('stale'))).toBe(true); + expect(client1.listNamespacesWithMetadata).toHaveBeenCalledWith( + { stale: { title: 'string' } }, + ['stale'] + ); + }); + + it('passes all namespace keys for stale warnings including description-only declarations', async () => { + const client1 = { + listNamespacesWithMetadata: vi.fn().mockResolvedValue({ + namespaces: [ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + schema_source: 'sampled', + }, + ], + warnings: [ + 'Declared namespace "desc_only_stale" not found in Pinecone index "idx-a" — schema declaration is stale.', + ], + }), + checkIndexes: vi.fn().mockResolvedValue({ ok: true, errors: [] }), + getSparseIndexName: () => 'idx-a-sparse', + }; + const registry = buildSourceRegistry({ + sources: [ + { + ...sources[0]!, + namespaces: { + wg21: { description: 'Live namespace hint' }, + desc_only_stale: { description: 'Stale namespace hint' }, + }, + }, + ], + defaultSource: 'api_key_1', + cacheTtlMs: 60_000, + defaultTopK: 10, + requestTimeoutMs: 15_000, + clients: new Map([['api_key_1', client1 as never]]), + }); + const result = await registry.getNamespacesWithCache('api_key_1'); + expect(client1.listNamespacesWithMetadata).toHaveBeenCalledWith(undefined, [ + 'wg21', + 'desc_only_stale', + ]); + expect(result.data[0]?.description).toBe('Live namespace hint'); + expect(result.warnings?.some((w) => w.includes('desc_only_stale'))).toBe(true); + }); }); diff --git a/src/core/server/source-registry.ts b/src/core/server/source-registry.ts index a45a487..9a05a08 100644 --- a/src/core/server/source-registry.ts +++ b/src/core/server/source-registry.ts @@ -4,17 +4,16 @@ import { PineconeClient } from '../pinecone-client.js'; import type { NamespaceInfo } from './server-context.js'; +import { fetchNamespacesWithDeclaredConfig, type NamespacesCacheEntry } from './namespace-cache.js'; import type { SourceDefinition } from './source-config.js'; -type CacheEntry = { - data: NamespaceInfo[]; - expiresAt: number; -}; +export type { NamespacesCacheEntry }; export type PerSourceCacheResult = { data: NamespaceInfo[]; cache_hit: boolean; expires_at: number; + warnings?: string[]; }; export type AggregatedCacheResult = { @@ -22,6 +21,7 @@ export type AggregatedCacheResult = { cache_hit: boolean; expires_at: number; source_errors?: Record; + warnings?: string[]; }; export type BuildSourceRegistryOptions = { @@ -36,7 +36,7 @@ export type BuildSourceRegistryOptions = { export class SourceRegistry { private readonly entries: Map< string, - { client: PineconeClient; cache: CacheEntry | null; definition: SourceDefinition } + { client: PineconeClient; cache: NamespacesCacheEntry | null; definition: SourceDefinition } >; private readonly defaultSourceName: string; private readonly cacheTtlMs: number; @@ -106,18 +106,22 @@ export class SourceRegistry { data: entry.cache.data, cache_hit: true, expires_at: entry.cache.expiresAt, + ...(entry.cache.warnings.length > 0 ? { warnings: [...entry.cache.warnings] } : {}), }; } - const raw = await entry.client.listNamespacesWithMetadata(); - const data: NamespaceInfo[] = raw.map((ns) => ({ - namespace: ns.namespace, - recordCount: ns.recordCount, - metadata: ns.metadata, - source, - })); + const { data, warnings } = await fetchNamespacesWithDeclaredConfig( + entry.client, + entry.definition.namespaces, + source + ); const expiresAt = now + this.cacheTtlMs; - entry.cache = { data, expiresAt }; - return { data, cache_hit: false, expires_at: expiresAt }; + entry.cache = { data, expiresAt, warnings }; + return { + data, + cache_hit: false, + expires_at: expiresAt, + ...(warnings.length > 0 ? { warnings: [...warnings] } : {}), + }; } async getAllNamespacesWithCache(): Promise { @@ -130,6 +134,7 @@ export class SourceRegistry { ); const data: NamespaceInfo[] = []; const source_errors: Record = {}; + const warnings: string[] = []; let cache_hit = true; let maxExpires = 0; for (let i = 0; i < settled.length; i++) { @@ -140,6 +145,9 @@ export class SourceRegistry { if (!outcome.value.result.cache_hit) { cache_hit = false; } + if (outcome.value.result.warnings?.length) { + warnings.push(...outcome.value.result.warnings); + } maxExpires = Math.max(maxExpires, outcome.value.result.expires_at); } else { cache_hit = false; @@ -154,6 +162,7 @@ export class SourceRegistry { cache_hit, expires_at, ...(Object.keys(source_errors).length > 0 ? { source_errors } : {}), + ...(warnings.length > 0 ? { warnings } : {}), }; } diff --git a/src/core/server/tools/guided-query-tool.context.test.ts b/src/core/server/tools/guided-query-tool.context.test.ts index 520e406..976742a 100644 --- a/src/core/server/tools/guided-query-tool.context.test.ts +++ b/src/core/server/tools/guided-query-tool.context.test.ts @@ -11,6 +11,7 @@ import { expectMatchesResponseSchema, makeHybridQueryResult, makeSearchResult, + mockNamespacesWithMetadataResult, parseToolJson, } from './test-helpers.js'; @@ -26,7 +27,11 @@ function papersNamespaceClient(overrides?: { query?: ReturnType }) return { listNamespacesWithMetadata: vi .fn() - .mockResolvedValue([{ namespace: 'papers', recordCount: 42, metadata: namespaceMetadata }]), + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'papers', recordCount: 42, metadata: namespaceMetadata }, + ]) + ), query: overrides?.query ?? vi.fn().mockResolvedValue(makeHybridQueryResult()), count: vi.fn().mockResolvedValue({ count: 7, truncated: false }), }; @@ -34,19 +39,21 @@ function papersNamespaceClient(overrides?: { query?: ReturnType }) describe('guided_query tool handler (ServerContext instance path)', () => { it('returns success with decision_trace using injected context', async () => { - const listNamespacesWithMetadata = vi.fn().mockResolvedValue([ - { - namespace: 'papers', - recordCount: 42, - metadata: { - document_number: 'string', - title: 'string', - url: 'string', - author: 'string', - chunk_text: 'string', + const listNamespacesWithMetadata = vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'papers', + recordCount: 42, + metadata: { + document_number: 'string', + title: 'string', + url: 'string', + author: 'string', + chunk_text: 'string', + }, }, - }, - ]); + ]) + ); const query = vi.fn().mockResolvedValue(makeHybridQueryResult()); const ctx = createTestServerContext({ client: { @@ -129,18 +136,20 @@ describe('guided_query tool handler (ServerContext instance path)', () => { ); const ctx = createTestServerContext({ client: { - listNamespacesWithMetadata: vi.fn().mockResolvedValue([ - { - namespace: 'mailing', - recordCount: 42, - metadata: { - document_number: 'string', - title: 'string', - author: 'string', - chunk_text: 'string', + listNamespacesWithMetadata: vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'mailing', + recordCount: 42, + metadata: { + document_number: 'string', + title: 'string', + author: 'string', + chunk_text: 'string', + }, }, - }, - ]), + ]) + ), query, count: vi.fn(), } as never, diff --git a/src/core/server/tools/list-namespaces-tool.context.test.ts b/src/core/server/tools/list-namespaces-tool.context.test.ts index c02c0f2..f5c25ca 100644 --- a/src/core/server/tools/list-namespaces-tool.context.test.ts +++ b/src/core/server/tools/list-namespaces-tool.context.test.ts @@ -7,18 +7,21 @@ import { createTestServerContext, expectMatchesResponseSchema, makeMockPineconeClient, + mockNamespacesWithMetadataResult, parseToolJson, } from './test-helpers.js'; describe('list_namespaces tool handler (ServerContext instance path)', () => { it('returns namespaces from injected context cache miss', async () => { - const listNamespacesWithMetadata = vi.fn().mockResolvedValue([ - { - namespace: 'wg21', - recordCount: 10, - metadata: { title: 'string', url: 'string' }, - }, - ]); + const listNamespacesWithMetadata = vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string', url: 'string' }, + }, + ]) + ); const ctx = createTestServerContext({ client: { listNamespacesWithMetadata } as never, }); @@ -38,19 +41,22 @@ describe('list_namespaces tool handler (ServerContext instance path)', () => { name: 'wg21', record_count: 10, metadata_fields: { title: 'string', url: 'string' }, + schema_source: 'sampled', }, ]); expect(listNamespacesWithMetadata).toHaveBeenCalledOnce(); }); it('serves cached namespaces on second call via injected context', async () => { - const listNamespacesWithMetadata = vi.fn().mockResolvedValue([ - { - namespace: 'wg21', - recordCount: 10, - metadata: { title: 'string' }, - }, - ]); + const listNamespacesWithMetadata = vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + }, + ]) + ); const ctx = createTestServerContext({ client: { listNamespacesWithMetadata } as never, }); @@ -92,4 +98,119 @@ describe('list_namespaces tool handler (multi-source)', () => { expect(body['source_errors']).toEqual({ api_key_2: 'api_key_2 unreachable' }); expect(body['cache_hit']).toBe(false); }); + + it('surfaces schema_source and config_warnings when declarations mismatch live data', async () => { + const client1 = { + listNamespacesWithMetadata: vi.fn().mockResolvedValue({ + namespaces: [ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + schema_source: 'declared', + }, + ], + warnings: [ + 'Declared namespace "stale_ns" not found in Pinecone index "idx-a" — schema declaration is stale.', + ], + }), + query: vi.fn(), + count: vi.fn(), + keywordSearch: vi.fn(), + checkIndexes: vi.fn().mockResolvedValue({ ok: true, errors: [] }), + getSparseIndexName: () => 'sparse', + }; + const { ctx } = createMultiSourceTestContext({ + clients: new Map([['api_key_1', client1 as never]]), + sources: [ + { + name: 'api_key_1', + apiKey: 'k1', + indexName: 'idx-a', + namespaces: { + wg21: { metadata_schema: { title: 'string' } }, + stale_ns: { metadata_schema: { title: 'string' } }, + }, + }, + { + name: 'api_key_2', + apiKey: 'k2', + indexName: 'idx-b', + }, + ], + }); + + const server = createMockServer(); + registerListNamespacesTool(server as never, ctx); + const body = parseToolJson( + await server.getHandler('list_namespaces')!({ source: 'api_key_1' }) + ); + expect(body['namespaces']).toEqual([ + { + name: 'wg21', + record_count: 10, + metadata_fields: { title: 'string' }, + source: 'api_key_1', + schema_source: 'declared', + }, + ]); + expect(body['config_warnings']).toEqual([ + 'Declared namespace "stale_ns" not found in Pinecone index "idx-a" — schema declaration is stale.', + ]); + }); + + it('surfaces per-namespace description from private config on matching live rows', async () => { + const client1 = { + listNamespacesWithMetadata: vi.fn().mockResolvedValue({ + namespaces: [ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + schema_source: 'sampled', + }, + ], + warnings: [], + }), + query: vi.fn(), + count: vi.fn(), + keywordSearch: vi.fn(), + checkIndexes: vi.fn().mockResolvedValue({ ok: true, errors: [] }), + getSparseIndexName: () => 'sparse', + }; + const { ctx } = createMultiSourceTestContext({ + clients: new Map([['api_key_1', client1 as never]]), + sources: [ + { + name: 'api_key_1', + apiKey: 'k1', + indexName: 'idx-a', + namespaces: { + wg21: { description: 'WG21 papers corpus' }, + }, + }, + { + name: 'api_key_2', + apiKey: 'k2', + indexName: 'idx-b', + }, + ], + }); + + const server = createMockServer(); + registerListNamespacesTool(server as never, ctx); + const body = parseToolJson( + await server.getHandler('list_namespaces')!({ source: 'api_key_1' }) + ); + expect(body['namespaces']).toEqual([ + { + name: 'wg21', + record_count: 10, + metadata_fields: { title: 'string' }, + source: 'api_key_1', + schema_source: 'sampled', + description: 'WG21 papers corpus', + }, + ]); + }); }); diff --git a/src/core/server/tools/list-namespaces-tool.test.ts b/src/core/server/tools/list-namespaces-tool.test.ts index c925eb3..8fc559b 100644 --- a/src/core/server/tools/list-namespaces-tool.test.ts +++ b/src/core/server/tools/list-namespaces-tool.test.ts @@ -78,4 +78,38 @@ describe('list_namespaces tool handler', () => { const err = assertToolErrorCode(raw, 'PINECONE_ERROR'); expect(err.message).toBe('Failed to list namespaces'); }); + + it('propagates config_warnings and schema_source via legacy namespaces-cache facade', async () => { + mockedGetNamespaces.mockResolvedValue({ + data: [ + { + namespace: 'wg21', + recordCount: 10, + metadata: { title: 'string' }, + schema_source: 'declared', + }, + ], + cache_hit: false, + expires_at: Date.now() + 60_000, + warnings: [ + 'Declared namespace "stale_ns" not found in Pinecone index "idx-a" — schema declaration is stale.', + ], + }); + + const server = createMockServer(); + registerListNamespacesTool(server as never); + const body = parseToolJson(await server.getHandler('list_namespaces')!({})); + + expect(body.config_warnings).toEqual([ + 'Declared namespace "stale_ns" not found in Pinecone index "idx-a" — schema declaration is stale.', + ]); + expect(body.namespaces).toEqual([ + { + name: 'wg21', + record_count: 10, + metadata_fields: { title: 'string' }, + schema_source: 'declared', + }, + ]); + }); }); diff --git a/src/core/server/tools/list-namespaces-tool.ts b/src/core/server/tools/list-namespaces-tool.ts index b9855ef..abc3212 100644 --- a/src/core/server/tools/list-namespaces-tool.ts +++ b/src/core/server/tools/list-namespaces-tool.ts @@ -37,6 +37,9 @@ async function executeListNamespaces(source: string | undefined, ctx?: ServerCon typeof rawSourceErrors === 'object' ? (rawSourceErrors as Record) : undefined; + const rawWarnings = 'warnings' in cacheResult ? cacheResult.warnings : undefined; + const config_warnings = + Array.isArray(rawWarnings) && rawWarnings.length > 0 ? rawWarnings : undefined; const now = Date.now(); const ttlSeconds = Math.max(0, Math.floor((expires_at - now) / 1000)); @@ -47,11 +50,14 @@ async function executeListNamespaces(source: string | undefined, ctx?: ServerCon expires_at_iso: new Date(expires_at).toISOString(), count: namespacesInfo.length, ...(source_errors !== undefined && source_errors !== null ? { source_errors } : {}), + ...(config_warnings !== undefined ? { config_warnings } : {}), namespaces: namespacesInfo.map((ns) => ({ name: ns.namespace, record_count: ns.recordCount, metadata_fields: ns.metadata, ...(ns.source !== undefined ? { source: ns.source } : {}), + ...(ns.schema_source !== undefined ? { schema_source: ns.schema_source } : {}), + ...(ns.description !== undefined ? { description: ns.description } : {}), })), }; diff --git a/src/core/server/tools/list-sources-tool.test.ts b/src/core/server/tools/list-sources-tool.test.ts new file mode 100644 index 0000000..b3ebd29 --- /dev/null +++ b/src/core/server/tools/list-sources-tool.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { registerListSourcesTool } from './list-sources-tool.js'; +import { listSourcesResponseSchema } from '../response-schemas.js'; +import { + assertToolErrorCode, + createMockServer, + createMultiSourceTestContext, + expectMatchesResponseSchema, + parseToolJson, +} from './test-helpers.js'; + +describe('list_sources tool handler', () => { + it('returns LIFECYCLE when not in multi-source mode', async () => { + const server = createMockServer(); + registerListSourcesTool(server as never); + const err = assertToolErrorCode(await server.getHandler('list_sources')!({}), 'LIFECYCLE'); + expect(err.message).toMatch(/multi-source/i); + }); + + it('returns sources without description when none configured (back-compat shape)', async () => { + const { ctx } = createMultiSourceTestContext(); + const server = createMockServer(); + registerListSourcesTool(server as never, ctx); + const body = parseToolJson(await server.getHandler('list_sources')!({})); + expectMatchesResponseSchema(listSourcesResponseSchema, body); + expect(body['sources']).toEqual([{ name: 'api_key_1' }, { name: 'api_key_2' }]); + expect(body['default']).toBe('api_key_1'); + }); + + it('returns configured per-source description when present', async () => { + const { ctx } = createMultiSourceTestContext({ + sources: [ + { + name: 'api_key_1', + apiKey: 'k1', + indexName: 'idx-a', + description: 'Public corpus', + }, + { + name: 'api_key_2', + apiKey: 'k2', + indexName: 'idx-b', + }, + ], + }); + const server = createMockServer(); + registerListSourcesTool(server as never, ctx); + const body = parseToolJson(await server.getHandler('list_sources')!({})); + expect(body['sources']).toEqual([ + { name: 'api_key_1', description: 'Public corpus' }, + { name: 'api_key_2' }, + ]); + }); +}); diff --git a/src/core/server/tools/list-sources-tool.ts b/src/core/server/tools/list-sources-tool.ts index 99a5641..b13fa78 100644 --- a/src/core/server/tools/list-sources-tool.ts +++ b/src/core/server/tools/list-sources-tool.ts @@ -11,7 +11,7 @@ export function registerListSourcesTool(server: McpServer, ctx?: ServerContext): { description: 'List configured Pinecone source names when multiple API keys/projects are active. ' + - 'Returns source ids and the default source.', + 'Returns source ids, optional per-source descriptions from private config, and the default source.', inputSchema: {}, }, async () => { @@ -26,7 +26,7 @@ export function registerListSourcesTool(server: McpServer, ctx?: ServerContext): } const response: ListSourcesResponse = { status: 'success', - sources: ctx.listSources(), + sources: ctx.listSourceDetails(), default: ctx.getDefaultSourceName(), }; return validatedJsonResponse(listSourcesResponseSchema, response); diff --git a/src/core/server/tools/namespace-router-tool.context.test.ts b/src/core/server/tools/namespace-router-tool.context.test.ts index c60e5d5..a570f4b 100644 --- a/src/core/server/tools/namespace-router-tool.context.test.ts +++ b/src/core/server/tools/namespace-router-tool.context.test.ts @@ -5,18 +5,21 @@ import { createMockServer, createTestServerContext, expectMatchesResponseSchema, + mockNamespacesWithMetadataResult, parseToolJson, } from './test-helpers.js'; describe('namespace_router tool handler (ServerContext instance path)', () => { it('returns ranked suggestions from injected context cache miss', async () => { - const listNamespacesWithMetadata = vi.fn().mockResolvedValue([ - { - namespace: 'papers', - recordCount: 42, - metadata: { title: 'string', document_number: 'string' }, - }, - ]); + const listNamespacesWithMetadata = vi.fn().mockResolvedValue( + mockNamespacesWithMetadataResult([ + { + namespace: 'papers', + recordCount: 42, + metadata: { title: 'string', document_number: 'string' }, + }, + ]) + ); const ctx = createTestServerContext({ client: { listNamespacesWithMetadata } as never, }); diff --git a/src/core/server/tools/test-helpers.ts b/src/core/server/tools/test-helpers.ts index cdcd1ab..be00f1a 100644 --- a/src/core/server/tools/test-helpers.ts +++ b/src/core/server/tools/test-helpers.ts @@ -174,6 +174,25 @@ export const DEFAULT_MULTI_SOURCE_DEFINITIONS: SourceDefinition[] = [ { name: 'api_key_2', apiKey: 'k2', indexName: 'idx-b', sparseIndexName: 'idx-b-sparse' }, ]; +/** Wrap namespace rows in the shape returned by `listNamespacesWithMetadata`. */ +export function mockNamespacesWithMetadataResult( + namespaces: Array<{ + namespace: string; + recordCount: number; + metadata: Record; + schema_source?: 'declared' | 'sampled'; + }>, + warnings: string[] = [] +) { + return { + namespaces: namespaces.map((ns) => ({ + ...ns, + schema_source: ns.schema_source ?? ('sampled' as const), + })), + warnings, + }; +} + /** Minimal mock {@link PineconeClient} with configurable namespace list. */ export function makeMockPineconeClient( namespaces: string[], @@ -181,11 +200,13 @@ export function makeMockPineconeClient( ) { return { listNamespacesWithMetadata: vi.fn().mockResolvedValue( - namespaces.map((namespace) => ({ - namespace, - recordCount: 1, - metadata: { title: 'string' }, - })) + mockNamespacesWithMetadataResult( + namespaces.map((namespace) => ({ + namespace, + recordCount: 1, + metadata: { title: 'string' }, + })) + ) ), query: options?.query ?? vi.fn(), count: vi.fn().mockResolvedValue({ count: 0, truncated: false }), diff --git a/src/core/setup-multi-instance.test.ts b/src/core/setup-multi-instance.test.ts index c153d62..1e0a4a4 100644 --- a/src/core/setup-multi-instance.test.ts +++ b/src/core/setup-multi-instance.test.ts @@ -7,6 +7,7 @@ import * as listSourcesTool from './server/tools/list-sources-tool.js'; import { createTestServerContext, isolateFromDefaultContext, + mockNamespacesWithMetadataResult, resolveTestConfig, } from './server/tools/test-helpers.js'; @@ -76,10 +77,18 @@ describe('setup multi-instance (phase 4)', () => { isolateFromDefaultContext(); const listNsA = vi .fn() - .mockResolvedValue([{ namespace: 'wg21', recordCount: 10, metadata: { source: 'a' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'wg21', recordCount: 10, metadata: { source: 'a' } }, + ]) + ); const listNsB = vi .fn() - .mockResolvedValue([{ namespace: 'other', recordCount: 5, metadata: { source: 'b' } }]); + .mockResolvedValue( + mockNamespacesWithMetadataResult([ + { namespace: 'other', recordCount: 5, metadata: { source: 'b' } }, + ]) + ); const cfgA = resolveTestConfig({ apiKey: 'cache-iso-a' }); const cfgB = resolveTestConfig({ apiKey: 'cache-iso-b' }); const ctxA = createTestServerContext({