From f0439eec7731b4aaea6d4a5780ae56fc064f30e3 Mon Sep 17 00:00:00 2001 From: zho Date: Sun, 15 Feb 2026 08:35:52 +0800 Subject: [PATCH 01/20] #8-update mcp with several tools --- .dockerignore | 11 + .github/workflows/ci.yml | 6 + CHANGELOG.md | 7 + Dockerfile | 27 ++ README.md | 222 +++++++++++++-- package.json | 5 +- src/config.ts | 5 + src/constants.ts | 17 +- src/logger.ts | 61 ++++ src/pinecone-client.test.ts | 106 +++++++ src/pinecone-client.ts | 264 ++++++++++++++---- src/server.test.ts | 80 ++++++ src/server.ts | 262 ++--------------- src/server/client-context.ts | 15 + src/server/metadata-filter.ts | 65 +++++ src/server/namespace-router.ts | 74 +++++ src/server/namespaces-cache.ts | 40 +++ src/server/query-suggestion.ts | 77 +++++ src/server/suggestion-flow.ts | 49 ++++ src/server/tool-response.ts | 27 ++ src/server/tools/count-tool.ts | 80 ++++++ src/server/tools/generate-urls-tool.ts | 64 +++++ src/server/tools/guided-query-tool.ts | 221 +++++++++++++++ src/server/tools/list-namespaces-tool.ts | 48 ++++ src/server/tools/namespace-router-tool.ts | 46 +++ src/server/tools/query-tool.ts | 205 ++++++++++++++ src/server/tools/suggest-query-params-tool.ts | 63 +++++ src/server/url-generation.test.ts | 58 ++++ src/server/url-generation.ts | 64 +++++ src/types.ts | 57 +++- tsconfig.json | 16 +- 31 files changed, 2000 insertions(+), 342 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 src/config.ts create mode 100644 src/logger.ts create mode 100644 src/server.test.ts create mode 100644 src/server/client-context.ts create mode 100644 src/server/metadata-filter.ts create mode 100644 src/server/namespace-router.ts create mode 100644 src/server/namespaces-cache.ts create mode 100644 src/server/query-suggestion.ts create mode 100644 src/server/suggestion-flow.ts create mode 100644 src/server/tool-response.ts create mode 100644 src/server/tools/count-tool.ts create mode 100644 src/server/tools/generate-urls-tool.ts create mode 100644 src/server/tools/guided-query-tool.ts create mode 100644 src/server/tools/list-namespaces-tool.ts create mode 100644 src/server/tools/namespace-router-tool.ts create mode 100644 src/server/tools/query-tool.ts create mode 100644 src/server/tools/suggest-query-params-tool.ts create mode 100644 src/server/url-generation.test.ts create mode 100644 src/server/url-generation.ts diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5ec97af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +node_modules +dist +.git +.github +.cursor +npm-debug.log +*.log +.env +.env.* +coverage +*.tsbuildinfo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d509fee..8a560ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,9 @@ jobs: - name: Build run: npm run build + - name: Smoke test CLI + run: npm run smoke + - name: Run tests run: npm test @@ -58,3 +61,6 @@ jobs: - name: Security audit run: npm audit --audit-level=moderate continue-on-error: true + + - name: Package dry run + run: npm pack --dry-run diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c3a6c..dfb39a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added + - Initial TypeScript implementation of Pinecone Read-Only MCP - Hybrid search support (dense + sparse embeddings) - Semantic reranking using BGE reranker model @@ -20,23 +21,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Complete documentation ### Changed + - N/A ### Deprecated + - N/A ### Removed + - N/A ### Fixed + - N/A ### Security + - N/A ## [0.1.0] - 2026-01-26 ### Added + - Initial release of TypeScript version - Feature parity with Python version - Production-ready implementation with: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1fcaac5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM node:20-bookworm-slim AS build + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci + +COPY . . +RUN npm run build + +FROM node:20-bookworm-slim AS runtime + +WORKDIR /app +ENV NODE_ENV=production + +# Create non-root runtime user +RUN useradd --create-home --uid 10001 mcpuser + +COPY package*.json ./ +RUN npm ci --omit=dev && npm cache clean --force + +COPY --from=build /app/dist ./dist +COPY README.md LICENSE CHANGELOG.md ./ + +USER mcpuser + +ENTRYPOINT ["node", "dist/index.js"] diff --git a/README.md b/README.md index d97caf5..013aa93 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,12 @@ The server requires a Pinecone API key and supports the following configuration ### Environment Variables -| Variable | Required | Default | Description | -|----------|----------|---------|-------------| -| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | -| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name | -| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | -| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | +| Variable | Required | Default | Description | +| ---------------------------------- | -------- | -------------------- | --------------------- | +| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | +| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name | +| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | +| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | ### Claude Desktop Configuration @@ -93,9 +93,12 @@ Or with explicit options: "args": [ "-y", "@will-cppa/pinecone-read-only-mcp", - "--api-key", "your-api-key-here", - "--index-name", "your-index-name", - "--rerank-model", "bge-reranker-v2-m3" + "--api-key", + "your-api-key-here", + "--index-name", + "your-index-name", + "--rerank-model", + "bge-reranker-v2-m3" ] } } @@ -147,6 +150,48 @@ node node_modules/@will-cppa/pinecone-read-only-mcp/dist/index.js --api-key YOUR --help, -h Show help message ``` +## Deployment + +### Production Readiness Defaults + +- Build now **fails fast** on TypeScript errors (`npm run build` no longer suppresses failures). +- CI validates typecheck, lint, format, build, smoke run, tests, and package dry-run. +- `list_namespaces` data is cached in-memory for 30 minutes to reduce repeated Pinecone calls. +- Query/count flow has guardrails (`suggest_query_params` before execution) to prevent wasteful calls. + +### Deploy with npm Package + +```bash +# install +npm i @will-cppa/pinecone-read-only-mcp + +# run +npx @will-cppa/pinecone-read-only-mcp --api-key YOUR_API_KEY +``` + +### Deploy with Docker + +```bash +# build image +docker build -t pinecone-read-only-mcp:latest . + +# run (stdio MCP server) +docker run --rm -i \ + -e PINECONE_API_KEY=YOUR_API_KEY \ + -e PINECONE_INDEX_NAME=rag-hybrid \ + pinecone-read-only-mcp:latest +``` + +### Release Gate (recommended) + +Before tagging/releasing: + +```bash +npm run release:check +``` + +This runs full CI-equivalent checks and validates publish contents with `npm pack --dry-run`. + ## API Documentation The server exposes the following tools via MCP: @@ -159,7 +204,11 @@ Discovers and lists all available namespaces in the configured Pinecone index, i **Returns:** JSON object with namespace details including available metadata fields +`metadata_fields` values represent inferred field types from sampled records. Common values include: +`string`, `number`, `boolean`, `string[]`, and `array`. + **Example response:** + ```json { "status": "success", @@ -186,29 +235,134 @@ Discovers and lists all available namespaces in the configured Pinecone index, i } ``` +### `suggest_query_params` + +Suggests which **fields** to request and which tool to use (`count`, `query_fast`, or `query_detailed`), based on the namespace’s schema (from `list_namespaces`) and the user’s natural language query. This is a mandatory flow step before `count`/`query` tools. + +**Parameters:** + +| Parameter | Type | Required | Description | +| ------------ | ------ | -------- | ----------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace to query (must match a name from `list_namespaces`) | +| `user_query` | string | Yes | User’s question or intent (e.g. "list papers by Lakos with titles", "how many papers by Wong?") | + +**Returns:** `suggested_fields` (only fields that exist in that namespace), `use_count_tool`, `recommended_tool`, `explanation`, and `namespace_found`. + +**Example response:** + +```json +{ + "status": "success", + "suggested_fields": ["document_number", "title", "url", "author"], + "use_count_tool": false, + "recommended_tool": "query_fast", + "explanation": "User asked for a list or browse; use minimal fields (no chunk_text) for smaller payload and cost.", + "namespace_found": true +} +``` + +Use `suggested_fields` as the `fields` parameter when calling query tools. + +### `guided_query` + +Single orchestrator tool that runs the full flow in one call: + +1. namespace routing (if namespace is omitted), +2. query param suggestion, +3. execution via `count`, `query_fast`, or `query_detailed`. + +It returns both the final result and a `decision_trace` for transparency. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +| ----------------- | ------- | -------- | ------- | ----------------------------------------------------------------------------------- | +| `user_query` | string | Yes | - | User question/intent | +| `namespace` | string | No | - | Optional explicit namespace | +| `metadata_filter` | object | No | - | Optional metadata filter | +| `top_k` | integer | No | `10` | Query result size for query paths (1-100) | +| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `query_fast`, `query_detailed` | +| `enrich_urls` | boolean | No | `true` | Auto-generate URLs for `mailing` and `slack-Cpplang` when `metadata.url` is missing | + +**Returns:** JSON containing `decision_trace` and `result`. + +### `generate_urls` + +Generates URLs for retrieved records when metadata does not contain `url` and URL is required. + +Supported namespaces: + +- `mailing` +- `slack-Cpplang` + +Rules: + +- **`mailing`**: uses `doc_id` or `thread_id` + Format: `https://lists.boost.org/archives/list/{doc_id_or_thread_id}/` +- **`slack-Cpplang`**: prefer `source` directly if present; otherwise use `team_id`, `channel_id`, and `doc_id` + `message_id = doc_id.replace('.', '')` + Format: `https://app.slack.com/{team_id}/{channel_id}/p{message_id}` + +**Parameters:** + +| Parameter | Type | Required | Description | +| ----------- | ------ | -------- | --------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace for URL-generation logic | +| `records` | array | Yes | Retrieved records; each item can be either metadata itself or an object with `metadata` field | + +**Returns:** Per-record generated URL, generation method, and reason if unavailable. + +### `count` + +Returns the **unique document count** matching a metadata filter and semantic query. Use for questions like "how many papers by Lakos?" instead of the `query` tool. For performance, the count tool uses **semantic (dense) search only** (no hybrid or lexical) and requests only document identifiers (`document_number`, `url`, `doc_id`)—no chunk content—then deduplicates by document. + +**Parameters:** + +| Parameter | Type | Required | Description | +| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace to count in (use `list_namespaces` to discover) | +| `query_text` | string | Yes | Search query; use a broad term (e.g. `"paper"`, `"document"`) when counting by metadata only | +| `metadata_filter` | object | No | Same operators as `query` (e.g. `{"author": {"$in": ["John Lakos"]}}` for wg21-papers) | + +**Returns:** JSON with `count` (unique documents, up to 10,000), and `truncated: true` if there are at least 10,000 matches. + +**Example response:** + +```json +{ + "status": "success", + "count": 42, + "truncated": false, + "namespace": "wg21-papers", + "metadata_filter": { "author": { "$in": ["John Lakos"] } } +} +``` + ### `query` -Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. +Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. For **count** questions, use the `count` tool instead. **Parameters:** -| Parameter | Type | Required | Default | Description | -|-----------|------|----------|---------|-------------| -| `query_text` | string | Yes | - | Search query text | -| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | -| `top_k` | integer | No | `10` | Number of results (1-100) | -| `use_reranking` | boolean | No | `true` | Enable semantic reranking | -| `metadata_filter` | object | No | - | Metadata filter to narrow results (e.g., `{"author": "John", "year": 2023}`) | +| Parameter | Type | Required | Default | Description | +| ----------------- | -------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `query_text` | string | Yes | - | Search query text | +| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | +| `top_k` | integer | No | `10` | Number of results (1-100) | +| `use_reranking` | boolean | No | `true` | Enable semantic reranking | +| `metadata_filter` | object | No | - | Metadata filter to narrow results (e.g., `{"author": "John", "year": 2023}`) | +| `fields` | string[] | No | - | Field names to return (e.g. `["document_number", "title", "url"]`). Omit for all fields; include `chunk_text` for content. Reduces payload and cost. | -**Returns:** JSON object with search results including content, relevance scores, and metadata +**Returns:** JSON object with search results (only requested fields when `fields` is set), relevance scores, and metadata **Example response:** + ```json { "status": "success", "query": "your search query", "namespace": "namespace1", - "metadata_filter": {"author": "John Doe"}, + "metadata_filter": { "author": "John Doe" }, "result_count": 10, "results": [ { @@ -230,16 +384,16 @@ Metadata filters allow you to narrow down search results based on document prope **Supported Operators (10 total):** -| Operator | Syntax | Description | Example | -|----------|--------|-------------|---------| -| Equal | `$eq` or value directly | Exact match | `{"status": "published"}` or `{"status": {"$eq": "published"}}` | -| Not Equal | `$ne` | Not equal to | `{"status": {"$ne": "draft"}}` | -| Greater Than | `$gt` | Greater than | `{"year": {"$gt": 2022}}` | -| Greater Than or Equal | `$gte` | Greater than or equal | `{"timestamp": {"$gte": 1704067200}}` | -| Less Than | `$lt` | Less than | `{"score": {"$lt": 0.5}}` | -| Less Than or Equal | `$lte` | Less than or equal | `{"priority": {"$lte": 3}}` | -| In Array | `$in` | Value is in array field | `{"tags": {"$in": ["cpp", "contracts"]}}` (only for array-type fields) | -| Not In Array | `$nin` | Value not in array field | `{"tags": {"$nin": ["draft", "archived"]}}` (only for array-type fields) | +| Operator | Syntax | Description | Example | +| --------------------- | ----------------------- | ------------------------ | ------------------------------------------------------------------------ | +| Equal | `$eq` or value directly | Exact match | `{"status": "published"}` or `{"status": {"$eq": "published"}}` | +| Not Equal | `$ne` | Not equal to | `{"status": {"$ne": "draft"}}` | +| Greater Than | `$gt` | Greater than | `{"year": {"$gt": 2022}}` | +| Greater Than or Equal | `$gte` | Greater than or equal | `{"timestamp": {"$gte": 1704067200}}` | +| Less Than | `$lt` | Less than | `{"score": {"$lt": 0.5}}` | +| Less Than or Equal | `$lte` | Less than or equal | `{"priority": {"$lte": 3}}` | +| In Array | `$in` | Value is in array field | `{"tags": {"$in": ["cpp", "contracts"]}}` (only for array-type fields) | +| Not In Array | `$nin` | Value not in array field | `{"tags": {"$nin": ["draft", "archived"]}}` (only for array-type fields) | **Filter Examples:** @@ -275,11 +429,14 @@ Metadata filters allow you to narrow down search results based on document prope ``` **Important Limitations:** + - **String fields require EXACT match** - No wildcards, partial matches, or substring searches - **Comma-separated strings**: If a field contains `"John Lakos, Herb Sutter"`, you cannot filter for just `"John Lakos"` - You must match the entire string: `{"author": "John Lakos, Herb Sutter"}` - To filter by individual authors, the data must be stored as an array field - **`$in` and `$nin` operators**: Only work on array-type fields, not comma-separated strings +- **Unsupported operators are rejected**: Unknown operators (for example `$regex`) return a validation error +- **`$in` and `$nin` must use arrays**: `{"tags": {"$in": "cpp"}}` is invalid; use `{"tags": {"$in": ["cpp"]}}` - Multiple conditions at the top level are combined with **AND** logic - Use comparison operators (`$gt`, `$gte`, `$lt`, `$lte`) for numeric and timestamp fields - Direct value assignment implies `$eq` (exact match) @@ -354,12 +511,14 @@ npm run dev -- --api-key YOUR_API_KEY ## Dependencies ### Production Dependencies + - [@modelcontextprotocol/sdk](https://www.npmjs.com/package/@modelcontextprotocol/sdk) - MCP SDK for TypeScript - [@pinecone-database/pinecone](https://www.npmjs.com/package/@pinecone-database/pinecone) - Pinecone client SDK - [zod](https://www.npmjs.com/package/zod) - TypeScript-first schema validation - [dotenv](https://www.npmjs.com/package/dotenv) - Environment variable management ### Development Dependencies + - [TypeScript](https://www.typescriptlang.org/) - Type-safe JavaScript - [ESLint](https://eslint.org/) - Code linting - [Prettier](https://prettier.io/) - Code formatting @@ -380,12 +539,14 @@ This TypeScript implementation provides the same functionality as the [Python ve ### API Key Issues If you see "Pinecone API key is required" error: + 1. Ensure `PINECONE_API_KEY` environment variable is set, OR 2. Pass `--api-key` option when running the server ### Index Not Found If you see index-related errors: + 1. Verify your index name is correct 2. Ensure your API key has access to the index 3. Check that both `your-index-name` and `your-index-name-sparse` indexes exist @@ -393,6 +554,7 @@ If you see index-related errors: ### Connection Issues If you experience connection issues: + 1. Check your internet connection 2. Verify Pinecone service status 3. Ensure firewall/proxy settings allow connections to Pinecone @@ -408,6 +570,7 @@ This project is licensed under the Boost Software License 1.0 - see the [LICENSE ## Acknowledgements This project uses: + - [Pinecone](https://www.pinecone.io/) for vector storage and retrieval - [Model Context Protocol](https://modelcontextprotocol.io/) for standardized AI integration - Hybrid search approach combining dense embeddings with sparse BM25-style retrieval @@ -420,6 +583,7 @@ This project uses: ## Support For issues and questions: + - GitHub Issues: [https://github.com/iTinkerBell/pinecone-read-only-mcp-typescript/issues](https://github.com/iTinkerBell/pinecone-read-only-mcp-typescript/issues) - Email: will@cppalliance.org diff --git a/package.json b/package.json index c65b456..caf3f11 100644 --- a/package.json +++ b/package.json @@ -36,10 +36,12 @@ "node": ">=18.0.0" }, "scripts": { - "build": "node ./node_modules/typescript/bin/tsc || exit 0", + "clean": "node -e \"const fs=require('fs'); fs.rmSync('dist',{recursive:true,force:true});\"", + "build": "npm run clean && node ./node_modules/typescript/bin/tsc", "build:watch": "tsc --watch", "dev": "tsx watch src/index.ts", "start": "node dist/index.js", + "smoke": "npm run build && node dist/index.js --help", "test": "vitest run", "test:watch": "vitest", "test:search": "tsx scripts/test-search.ts", @@ -50,6 +52,7 @@ "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "typecheck": "tsc --noEmit", "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm test", + "release:check": "npm run ci && npm pack --dry-run", "ci:local": "bash scripts/ci-local.sh", "prepublishOnly": "npm run ci", "prepack": "npm run build" diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..47273f2 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,5 @@ +/** + * Shared runtime config types. + */ + +export type LogLevel = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; diff --git a/src/constants.ts b/src/constants.ts index ef5faa8..57227b9 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -7,6 +7,14 @@ export const DEFAULT_RERANK_MODEL = 'bge-reranker-v2-m3'; export const DEFAULT_TOP_K = 10; export const MAX_TOP_K = 100; export const MIN_TOP_K = 1; +/** Namespace and suggestion caches stay valid for 30 minutes. */ +export const FLOW_CACHE_TTL_MS = 30 * 60 * 1000; +/** Top-k used by the count tool to get total matching documents (Pinecone allows high values). */ +export const COUNT_TOP_K = 10_000; +/** Minimal fields requested for count (no chunk_text) to reduce payload and cost. */ +export const COUNT_FIELDS = ['document_number', 'url', 'doc_id'] as const; +/** Default lightweight field set for fast queries. */ +export const FAST_QUERY_FIELDS = ['document_number', 'title', 'url', 'author', 'doc_id'] as const; export const DEFAULT_NAMESPACE = 'mailing'; export const SERVER_NAME = 'Pinecone Read-Only MCP'; @@ -19,7 +27,12 @@ Features: - Semantic Reranking: Uses BGE reranker model for improved precision - Dynamic Namespace Discovery: Automatically discovers available namespaces - Metadata Filtering: Supports optional metadata filters for refined searches +- Namespace Router: Suggests likely namespace(s) from natural-language intent +- Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count. +- URL Generation: Use generate_urls to synthesize URLs for mailing/slack records when metadata lacks url. Usage: -1. Use list_namespaces to discover available namespaces in the index -2. Use query to perform semantic search with hybrid retrieval and reranking`; +1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index +2. Optionally use namespace_router to choose candidate namespace(s) from user intent +3. Call suggest_query_params before query/count tools (mandatory flow gate) to get suggested_fields and recommended tool +4. Use count for count questions, query_fast for lightweight retrieval, or query_detailed/query for content-heavy retrieval`; diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 0000000..d6b01b8 --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,61 @@ +/** + * Simple level-based logger for the MCP server. + * Logs to stderr; never logs secrets. Use for deploy/observability. + */ + +import type { LogLevel } from './config.js'; + +const LEVEL_ORDER: Record = { + DEBUG: 0, + INFO: 1, + WARN: 2, + ERROR: 3, +}; + +let currentLevel: LogLevel = 'INFO'; + +export function setLogLevel(level: LogLevel): void { + currentLevel = level; +} + +export function getLogLevel(): LogLevel { + return currentLevel; +} + +function shouldLog(level: LogLevel): boolean { + return LEVEL_ORDER[level] >= LEVEL_ORDER[currentLevel]; +} + +function formatMessage(level: string, msg: string, data?: unknown): string { + const ts = new Date().toISOString(); + const prefix = `[${ts}] [${level}]`; + if (data !== undefined) { + return `${prefix} ${msg} ${JSON.stringify(data)}`; + } + return `${prefix} ${msg}`; +} + +export function debug(msg: string, data?: unknown): void { + if (shouldLog('DEBUG')) { + console.error(formatMessage('DEBUG', msg, data)); + } +} + +export function info(msg: string, data?: unknown): void { + if (shouldLog('INFO')) { + console.error(formatMessage('INFO', msg, data)); + } +} + +export function warn(msg: string, data?: unknown): void { + if (shouldLog('WARN')) { + console.error(formatMessage('WARN', msg, data)); + } +} + +export function error(msg: string, err?: unknown): void { + if (shouldLog('ERROR')) { + const detail = err instanceof Error ? err.message : err !== undefined ? String(err) : undefined; + console.error(formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined)); + } +} diff --git a/src/pinecone-client.test.ts b/src/pinecone-client.test.ts index 01cea7f..8b9fa96 100644 --- a/src/pinecone-client.test.ts +++ b/src/pinecone-client.test.ts @@ -1,5 +1,19 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { PineconeClient } from './pinecone-client.js'; +import type { SearchableIndex, PineconeHit } from './types.js'; + +/** Test double: client with stubbable ensureIndexes and searchIndex for hybrid tests */ +type PineconeClientTestDouble = PineconeClient & { + ensureIndexes: () => Promise<{ denseIndex: SearchableIndex; sparseIndex: SearchableIndex }>; + searchIndex: ( + index: SearchableIndex, + query: string, + topK: number, + namespace?: string, + metadataFilter?: Record, + options?: { fields?: string[] } + ) => Promise; +}; describe('PineconeClient', () => { let client: PineconeClient; @@ -48,5 +62,97 @@ describe('PineconeClient', () => { }) ).rejects.toThrow('topK must be at least 1'); }); + + it('should continue hybrid search when one index fails', async () => { + const testClient = client as PineconeClientTestDouble; + + testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + + let searchCall = 0; + testClient.searchIndex = async () => { + searchCall += 1; + if (searchCall === 1) { + throw new Error('dense failure'); + } + return [ + { + _id: 'doc-1', + _score: 0.9, + fields: { chunk_text: 'hybrid content', author: 'tester' }, + }, + ]; + }; + + const results = await client.query({ + query: 'hybrid search', + namespace: 'test', + topK: 5, + useReranking: false, + }); + + expect(results).toHaveLength(1); + expect(results[0].content).toBe('hybrid content'); + expect(results[0].metadata.author).toBe('tester'); + }); + + it('should throw when both dense and sparse searches fail', async () => { + const testClient = client as PineconeClientTestDouble; + + testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + testClient.searchIndex = async () => { + throw new Error('index failure'); + }; + + await expect( + client.query({ + query: 'hybrid search', + namespace: 'test', + topK: 5, + useReranking: false, + }) + ).rejects.toThrow('Hybrid search failed: both dense and sparse index searches failed.'); + }); + }); + + describe('count', () => { + it('should return unique document count using semantic search only with minimal fields', async () => { + const testClient = client as PineconeClientTestDouble; + testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + + // Two chunks from doc A, one from doc B -> unique count 2 + testClient.searchIndex = async (_index, _query, _topK, _ns, _filter, options) => { + expect(options?.fields).toEqual(['document_number', 'url', 'doc_id']); + return [ + { _id: 'c1', _score: 1, fields: { document_number: 'p1234r0', url: 'https://example.com/1' } }, + { _id: 'c2', _score: 0.9, fields: { document_number: 'p1234r0', url: 'https://example.com/1' } }, + { _id: 'c3', _score: 0.8, fields: { document_number: 'p5678r0', url: 'https://example.com/2' } }, + ]; + }; + + const result = await client.count({ + query: 'paper', + namespace: 'wg21-papers', + metadataFilter: { author: { $in: ['John Doe'] } }, + }); + + expect(result.count).toBe(2); + expect(result.truncated).toBe(false); + }); + + it('should set truncated when hit limit is reached', async () => { + const testClient = client as PineconeClientTestDouble; + testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + const manyHits: PineconeHit[] = Array.from({ length: 10000 }, (_, i) => ({ + _id: `id-${i}`, + _score: 1, + fields: { doc_id: `doc-${i}` }, + })); + testClient.searchIndex = async () => manyHits; + + const result = await client.count({ query: 'paper', namespace: 'ns' }); + + expect(result.count).toBe(10000); + expect(result.truncated).toBe(true); + }); }); }); diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 6cf6bab..3490b1f 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -7,8 +7,43 @@ */ import { Pinecone } from '@pinecone-database/pinecone'; -import type { PineconeClientConfig, SearchResult, PineconeHit, QueryParams } from './types.js'; -import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL, DEFAULT_TOP_K, MAX_TOP_K } from './constants.js'; +import type { + PineconeClientConfig, + SearchResult, + PineconeHit, + QueryParams, + CountParams, + CountResult, + MergedHit, + SearchableIndex, + PineconeMetadataValue, +} from './types.js'; +import { + DEFAULT_INDEX_NAME, + DEFAULT_RERANK_MODEL, + DEFAULT_TOP_K, + MAX_TOP_K, + COUNT_TOP_K, + COUNT_FIELDS, +} from './constants.js'; + +/** + * Infers a human-readable metadata field type for namespace discovery. + * Distinguishes Pinecone-supported list type (string[]) from other arrays. + */ +function inferMetadataFieldType(value: unknown): string { + if (value === null || value === undefined) { + return 'unknown'; + } + if (Array.isArray(value)) { + if (value.length === 0) return 'array'; + if (value.every((item) => typeof item === 'string')) return 'string[]'; + return 'array'; + } + const t = typeof value; + if (t === 'string' || t === 'number' || t === 'boolean') return t; + return 'object'; +} export class PineconeClient { private apiKey: string; @@ -18,10 +53,8 @@ export class PineconeClient { // Lazy initialization private pc: Pinecone | null = null; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private denseIndex: any = null; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private sparseIndex: any = null; + private denseIndex: SearchableIndex | null = null; + private sparseIndex: SearchableIndex | null = null; private initialized = false; constructor(config: PineconeClientConfig) { @@ -52,9 +85,8 @@ export class PineconeClient { /** * Ensure Pinecone indexes are initialized and return them */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private async ensureIndexes(): Promise<{ denseIndex: any; sparseIndex: any }> { - if (this.initialized) { + private async ensureIndexes(): Promise<{ denseIndex: SearchableIndex; sparseIndex: SearchableIndex }> { + if (this.initialized && this.denseIndex !== null && this.sparseIndex !== null) { return { denseIndex: this.denseIndex, sparseIndex: this.sparseIndex }; } @@ -62,12 +94,14 @@ export class PineconeClient { const denseName = this.indexName; const sparseName = `${this.indexName}-sparse`; - this.denseIndex = pc.index(denseName); - this.sparseIndex = pc.index(sparseName); + const dense = pc.index(denseName) as unknown as SearchableIndex; + const sparse = pc.index(sparseName) as unknown as SearchableIndex; + this.denseIndex = dense; + this.sparseIndex = sparse; this.initialized = true; console.error(`Connected to indexes: ${denseName} and ${sparseName}`); - return { denseIndex: this.denseIndex, sparseIndex: this.sparseIndex }; + return { denseIndex: dense, sparseIndex: sparse }; } /** @@ -87,7 +121,7 @@ export class PineconeClient { const { denseIndex } = await this.ensureIndexes(); // Get index stats to find namespaces - const stats = await denseIndex.describeIndexStats(); + const stats = denseIndex.describeIndexStats ? await denseIndex.describeIndexStats() : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; console.error(`Found ${namespaces.length} namespace(s)`); @@ -96,27 +130,36 @@ export class PineconeClient { const namespacesInfo = await Promise.all( namespaces.map(async (ns: string) => { try { - const recordCount = stats.namespaces?.[ns]?.recordCount || 0; + const recordCount = stats?.namespaces?.[ns]?.recordCount || 0; const metadataFields: Record = {}; // Sample a few records to discover metadata fields - if (recordCount > 0) { + if (recordCount > 0 && denseIndex.namespace) { try { - const nsObj = denseIndex.namespace(ns); - // Query with a dummy vector to get some sample records - const sampleQuery = await nsObj.query({ - topK: 5, - vector: Array(stats.dimension || 1536).fill(0), - includeMetadata: true, - }); - - // Collect unique metadata fields and infer types + const nsObj = denseIndex.namespace(ns) as { query: (opts: { topK: number; vector: number[]; includeMetadata: boolean }) => Promise<{ matches?: Array<{ metadata?: Record }> }> } | null; + const sampleQuery = + nsObj && typeof nsObj.query === 'function' + ? await nsObj.query({ + topK: 5, + vector: Array((stats?.dimension ?? 1536)).fill(0), + includeMetadata: true, + }) + : { matches: undefined }; + + // Collect unique metadata fields and infer types (including string[]) if (sampleQuery?.matches) { - sampleQuery.matches.forEach((match: any) => { + sampleQuery.matches.forEach((match: { metadata?: Record }) => { if (match.metadata) { Object.entries(match.metadata).forEach(([key, value]) => { + const inferredType = inferMetadataFieldType(value); if (!(key in metadataFields)) { - metadataFields[key] = typeof value; + metadataFields[key] = inferredType; + } else if ( + metadataFields[key] === 'object' && + inferredType === 'string[]' + ) { + // Prefer array type over generic object when we see it in another sample + metadataFields[key] = inferredType; } }); } @@ -151,40 +194,66 @@ export class PineconeClient { } /** - * Search a Pinecone index using text query with optional metadata filtering + * Search a Pinecone index using text query with optional metadata filtering. + * When options.fields is set, only those fields are requested (e.g. for count: no chunk_text). */ private async searchIndex( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - index: any, + index: SearchableIndex, query: string, topK: number, namespace?: string, - metadataFilter?: Record + metadataFilter?: Record, + options?: { fields?: string[] } ): Promise { + // Build query payload in the same shape as Python implementation. + const queryPayload: Record = { + top_k: topK, + inputs: { text: query }, + }; + + // Include filter when explicitly provided (matches Python behavior). + if (metadataFilter !== undefined) { + queryPayload.filter = metadataFilter; + if (!options?.fields) { + console.error('Applying metadata filter:', JSON.stringify(metadataFilter)); + } + } + try { - // Get namespace object - const ns = namespace ? index.namespace(namespace) : index; + // Preferred path: Pinecone search API. + if (typeof index.search === 'function') { + const searchOpts: { namespace?: string; query: Record; fields?: string[] } = { + namespace, + query: queryPayload, + }; + if (options?.fields?.length) { + searchOpts.fields = options.fields; + } + const result = await index.search(searchOpts); + return result?.result?.hits || []; + } - // Use searchRecords API (Pinecone v5+) - const queryParams: any = { + // Backward-compatible fallback for older API shapes. + const target = namespace && index.namespace ? index.namespace(namespace) : index; + const queryParams: { query: Record; fields?: string[] } = { query: { topK, inputs: { text: query }, }, }; - - // Add metadata filter if provided - if (metadataFilter && Object.keys(metadataFilter).length > 0) { + if (metadataFilter !== undefined) { queryParams.query.filter = metadataFilter; - console.error('Applying metadata filter:', JSON.stringify(metadataFilter)); } - - const result = await ns.searchRecords(queryParams); - + if (options?.fields?.length) { + queryParams.fields = options.fields; + } + const result = target.searchRecords + ? await target.searchRecords(queryParams) + : { result: { hits: [] as PineconeHit[] } }; return result?.result?.hits || []; } catch (error) { - console.error('Error searching index:', error); - return []; + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error(`Pinecone search failed for namespace "${namespace ?? 'default'}": ${errorMessage}`); } } @@ -193,9 +262,8 @@ export class PineconeClient { * * Uses the higher score when duplicates are found. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private mergeResults(denseHits: PineconeHit[], sparseHits: PineconeHit[]): any[] { - const deduped: Record = {}; + private mergeResults(denseHits: PineconeHit[], sparseHits: PineconeHit[]): MergedHit[] { + const deduped: Record = {}; for (const hit of [...denseHits, ...sparseHits]) { const hitId = hit._id || ''; @@ -205,14 +273,14 @@ export class PineconeClient { continue; } - const hitMetadata: Record = {}; + const hitMetadata: Record = {}; let content = ''; for (const [key, value] of Object.entries(hit.fields || {})) { if (key === 'chunk_text') { - content = value; + content = typeof value === 'string' ? value : ''; } else { - hitMetadata[key] = value; + hitMetadata[key] = value as PineconeMetadataValue; } } @@ -232,8 +300,7 @@ export class PineconeClient { */ private async rerankResults( query: string, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - results: any[], + results: MergedHit[], topN: number ): Promise { if (!results || results.length === 0) { @@ -243,16 +310,21 @@ export class PineconeClient { const pc = this.ensureClient(); try { - const rerankResult = await pc.inference.rerank(this.rerankModel, query, results, { + const rerankResult = await pc.inference.rerank( + this.rerankModel, + query, + results as unknown as (string | Record)[], + { topN, rankFields: ['chunk_text'], returnDocuments: true, parameters: { truncate: 'END' }, - }); + } + ); const reranked: SearchResult[] = []; for (const item of rerankResult.data || []) { - const document = item.document || {}; + const document = (item.document || {}) as MergedHit; reranked.push({ id: document._id || '', content: document.chunk_text || '', @@ -282,7 +354,14 @@ export class PineconeClient { * and optionally reranks using the configured reranking model. */ async query(params: QueryParams): Promise { - const { query, topK: requestedTopK, namespace, metadataFilter, useReranking = true } = params; + const { + query, + topK: requestedTopK, + namespace, + metadataFilter, + useReranking = true, + fields: requestedFields, + } = params; // Validate inputs if (!query || !query.trim()) { @@ -293,19 +372,45 @@ export class PineconeClient { if (topK < 1) { throw new Error('topK must be at least 1'); } - if (topK > MAX_TOP_K) { - topK = MAX_TOP_K; // Cap at 100 for performance + // Allow up to COUNT_TOP_K when explicitly requested (e.g. for count tool); otherwise cap at MAX_TOP_K + const maxAllowed = + requestedTopK !== undefined && requestedTopK > MAX_TOP_K ? COUNT_TOP_K : MAX_TOP_K; + if (topK > maxAllowed) { + topK = maxAllowed; } + // When reranking, Pinecone requires chunk_text in returned fields; add it if user specified fields without it + const searchFields = + requestedFields?.length && + useReranking && + !requestedFields.includes('chunk_text') + ? [...requestedFields, 'chunk_text'] + : requestedFields; + // Ensure indexes are ready const { denseIndex, sparseIndex } = await this.ensureIndexes(); + const searchOptions = searchFields?.length ? { fields: searchFields } : undefined; + // Perform hybrid search - const [denseHits, sparseHits] = await Promise.all([ - this.searchIndex(denseIndex, query, topK, namespace, metadataFilter), - this.searchIndex(sparseIndex, query, topK, namespace, metadataFilter), + const [denseResult, sparseResult] = await Promise.allSettled([ + this.searchIndex(denseIndex, query, topK, namespace, metadataFilter, searchOptions), + this.searchIndex(sparseIndex, query, topK, namespace, metadataFilter, searchOptions), ]); + const denseHits = denseResult.status === 'fulfilled' ? denseResult.value : []; + const sparseHits = sparseResult.status === 'fulfilled' ? sparseResult.value : []; + + if (denseResult.status === 'rejected') { + console.error('Dense index search failed:', denseResult.reason); + } + if (sparseResult.status === 'rejected') { + console.error('Sparse index search failed:', sparseResult.reason); + } + if (denseResult.status === 'rejected' && sparseResult.status === 'rejected') { + throw new Error('Hybrid search failed: both dense and sparse index searches failed.'); + } + // Merge results const mergedResults = this.mergeResults(denseHits, sparseHits); @@ -329,4 +434,43 @@ export class PineconeClient { return documents; } + + /** + * Return the number of unique documents matching the query and optional metadata filter. + * Uses semantic search only (dense index), requests minimal fields (document_number, url, doc_id) + * to avoid transferring chunk content, and deduplicates by document for a document-level count. + */ + async count(params: CountParams): Promise { + const { denseIndex } = await this.ensureIndexes(); + + const hits = await this.searchIndex( + denseIndex, + params.query, + COUNT_TOP_K, + params.namespace, + params.metadataFilter, + { fields: [...COUNT_FIELDS] } + ); + + const docKeys = new Set(); + for (const hit of hits) { + const fields = hit.fields || {}; + const docNumber = fields.document_number; + const url = fields.url; + const docId = fields.doc_id; + const key = + (typeof docNumber === 'string' ? docNumber : undefined) ?? + (typeof url === 'string' ? url : undefined) ?? + (typeof docId === 'string' ? docId : undefined) ?? + hit._id ?? + ''; + docKeys.add(key); + } + + const count = docKeys.size; + return { + count, + truncated: hits.length >= COUNT_TOP_K, + }; + } } diff --git a/src/server.test.ts b/src/server.test.ts new file mode 100644 index 0000000..b431bf6 --- /dev/null +++ b/src/server.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest'; +import { validateMetadataFilter, suggestQueryParams } from './server.js'; + +describe('suggestQueryParams', () => { + const wg21Fields = { + author: 'string[]', + chunk_text: 'string', + document_number: 'string', + title: 'string', + type: 'string', + url: 'string', + }; + + it('suggests count tool and minimal fields for count-style queries', () => { + const r = suggestQueryParams(wg21Fields, 'How many papers by Lakos?'); + expect(r.use_count_tool).toBe(true); + expect(r.suggested_fields).toContain('document_number'); + expect(r.suggested_fields).toContain('url'); + expect(r.namespace_found).toBe(true); + }); + + it('suggests chunk_text for content-style queries', () => { + const r = suggestQueryParams(wg21Fields, 'What does the paper say about contracts?'); + expect(r.use_count_tool).toBe(false); + expect(r.suggested_fields).toContain('chunk_text'); + expect(r.namespace_found).toBe(true); + }); + + it('suggests minimal fields for list-style queries', () => { + const r = suggestQueryParams(wg21Fields, 'List papers by Michael Wong with titles and links'); + expect(r.use_count_tool).toBe(false); + expect(r.suggested_fields).not.toContain('chunk_text'); + expect(r.suggested_fields).toContain('title'); + expect(r.suggested_fields).toContain('url'); + expect(r.namespace_found).toBe(true); + }); + + it('returns namespace_found false when metadata is null', () => { + const r = suggestQueryParams(null, 'list papers'); + expect(r.namespace_found).toBe(false); + expect(r.suggested_fields).toEqual([]); + }); +}); + +describe('validateMetadataFilter', () => { + it('accepts direct scalar equality filters', () => { + const result = validateMetadataFilter({ + status: 'published', + year: 2024, + featured: true, + }); + + expect(result).toBeNull(); + }); + + it('accepts supported comparison operators and array operators', () => { + const result = validateMetadataFilter({ + year: { $gte: 2020, $lte: 2026 }, + tags: { $in: ['cpp', 'contracts'] }, + }); + + expect(result).toBeNull(); + }); + + it('rejects unsupported operators', () => { + const result = validateMetadataFilter({ + year: { $regex: '^202' }, + }); + + expect(result).toContain('Unsupported filter operator'); + }); + + it('rejects non-array values for $in/$nin', () => { + const result = validateMetadataFilter({ + tags: { $in: 'cpp' }, + }); + + expect(result).toContain('must use an array of primitive values'); + }); +}); diff --git a/src/server.ts b/src/server.ts index 0565aa7..e44f0eb 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,52 +1,24 @@ /** * Pinecone Read-Only MCP Server * - * This module implements a Model Context Protocol (MCP) server that provides - * semantic search capabilities over Pinecone vector databases using hybrid - * search (dense + sparse) with reranking. + * Keeps server composition slim by delegating validation, heuristics, + * and tool registration into dedicated modules. */ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { z } from 'zod'; -import { PineconeClient } from './pinecone-client.js'; -import { - SERVER_NAME, - SERVER_VERSION, - SERVER_INSTRUCTIONS, - MIN_TOP_K, - MAX_TOP_K, -} from './constants.js'; -import type { QueryResponse } from './types.js'; - -// Recursive Zod schema for Pinecone metadata filters -// Supports nested objects with operators like {"timestamp": {"$gte": 123}} -// Using z.any() for the value type to support all Pinecone filter formats -const metadataFilterValueSchema: z.ZodType = z.lazy(() => - z.union([ - z.string(), - z.number(), - z.boolean(), - z.array(z.string()), - z.array(z.number()), - z.record(z.string(), metadataFilterValueSchema), // Recursive for nested operators - ]) -); - -const metadataFilterSchema = z.record(z.string(), metadataFilterValueSchema); - -// Global Pinecone client (initialized lazily) -let pineconeClient: PineconeClient | null = null; - -function getPineconeClient(): PineconeClient { - if (!pineconeClient) { - throw new Error('Pinecone client not initialized. Call setPineconeClient first.'); - } - return pineconeClient; -} - -export function setPineconeClient(client: PineconeClient): void { - pineconeClient = client; -} +import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js'; +import { registerCountTool } from './server/tools/count-tool.js'; +import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js'; +import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js'; +import { registerListNamespacesTool } from './server/tools/list-namespaces-tool.js'; +import { registerNamespaceRouterTool } from './server/tools/namespace-router-tool.js'; +import { registerQueryTool } from './server/tools/query-tool.js'; +import { registerSuggestQueryParamsTool } from './server/tools/suggest-query-params-tool.js'; + +export { setPineconeClient } from './server/client-context.js'; +export { validateMetadataFilter } from './server/metadata-filter.js'; +export { suggestQueryParams } from './server/query-suggestion.js'; +export type { SuggestQueryParamsResult } from './server/query-suggestion.js'; export async function setupServer(): Promise { const server = new McpServer( @@ -59,203 +31,13 @@ export async function setupServer(): Promise { } ); - // Tool: list_namespaces - server.registerTool( - 'list_namespaces', - { - description: - 'List all available namespaces in the Pinecone index with their metadata fields and record counts. ' + - 'Returns detailed information about each namespace including available metadata fields that can be used for filtering in queries. ' + - 'Use this tool first to discover which namespaces exist and what metadata fields are available for filtering.', - inputSchema: {}, - }, - async () => { - try { - const client = getPineconeClient(); - const namespacesInfo = await client.listNamespacesWithMetadata(); - - const response = { - status: 'success', - count: namespacesInfo.length, - namespaces: namespacesInfo.map((ns) => ({ - name: ns.namespace, - record_count: ns.recordCount, - metadata_fields: ns.metadata, - })), - }; - - return { - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('Error listing namespaces:', error); - - const response = { - status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? errorMessage : 'Failed to list namespaces', - }; - - return { - isError: true, - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } - } - ); - - // Tool: query - server.registerTool( - 'query', - { - description: - 'Search the Pinecone vector database using hybrid semantic search with optional metadata filtering. ' + - 'Performs a hybrid search combining dense and sparse embeddings for better recall, with optional reranking for improved precision. ' + - 'Supports natural language queries and returns the most relevant documents based on semantic similarity. ' + - 'IMPORTANT: Use metadata_filter to narrow results. First call list_namespaces to discover available metadata fields. ' + - 'Metadata filters support 10 operators: $eq/==, $ne/!=, $gt/>, $gte/>=, $lt/<, $lte/<=, $in (array fields only), $nin (array fields only). ' + - 'NOTE: String fields require exact match - no wildcards. For comma-separated strings, you cannot filter by individual values. ' + - 'Use comparison operators for numeric/timestamp fields. Multiple top-level conditions use AND logic.', - inputSchema: { - query_text: z.string().describe('Search query text. Be specific for better results.'), - namespace: z - .string() - .describe( - 'Namespace to search within. Use list_namespaces tool to discover available namespaces in the index.' - ), - top_k: z - .number() - .int() - .min(MIN_TOP_K) - .max(MAX_TOP_K) - .default(10) - .describe('Number of results to return (1-100). Default: 10'), - use_reranking: z - .boolean() - .default(true) - .describe( - 'Whether to use semantic reranking for better relevance. Slower but more accurate. Default: true' - ), - metadata_filter: z - .record(z.string(), metadataFilterSchema) - .optional() - .describe( - 'Optional metadata filter to narrow down search results. Use exact field names from list_namespaces. ' + - 'Supports 10 operators: ' + - '$eq (==), $ne (!=), $gt (>), $gte (>=), $lt (<), $lte (<=), $in (array field contains value), $nin (array field not contains). ' + - 'IMPORTANT: String fields require EXACT match - no wildcards/partial matches. For comma-separated values (like authors), use exact full string. ' + - 'Examples: ' + - '{"author": "John Lakos"} - exact author match (single author only), ' + - '{"year": {"$gte": 2023}} - year >= 2023, ' + - '{"timestamp": {"$gt": 1704067200, "$lt": 1735689600}} - timestamp range, ' + - '{"status": "published"} - exact match (implicit $eq), ' + - '{"tags": {"$in": ["cpp", "contracts"]}} - tags array contains value (only works if tags is array field). ' + - 'Multiple conditions at top level are combined with AND logic.' - ), - }, - }, - async (params) => { - try { - const { query_text, namespace, top_k = 10, use_reranking = true, metadata_filter } = params; - - // Validate query - if (!query_text || !query_text.trim()) { - const response: QueryResponse = { - status: 'error', - message: 'Query text cannot be empty', - }; - - return { - isError: true, - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } - - // Log filter for debugging - if (metadata_filter) { - console.error('Received metadata filter:', JSON.stringify(metadata_filter, null, 2)); - } - - const client = getPineconeClient(); - const results = await client.query({ - query: query_text.trim(), - topK: top_k, - namespace, - useReranking: use_reranking, - metadataFilter: metadata_filter, - }); - - // Format results for output - const formattedResults = results.map((doc) => ({ - paper_number: - doc.metadata.document_number || - (doc.metadata.filename as string)?.replace('.md', '').toUpperCase() || - null, - title: doc.metadata.title || '', - author: doc.metadata.author || '', - url: doc.metadata.url || '', - content: doc.content.substring(0, 2000), // Truncate for readability - score: Math.round(doc.score * 10000) / 10000, - reranked: doc.reranked, - metadata: doc.metadata, // Include all metadata fields (including timestamp) - })); - - const response: QueryResponse = { - status: 'success', - query: query_text, - namespace, - metadata_filter: metadata_filter, - result_count: formattedResults.length, - results: formattedResults, - }; - - return { - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('Error executing query:', error); - - const response: QueryResponse = { - status: 'error', - message: - process.env.LOG_LEVEL === 'DEBUG' - ? errorMessage - : 'An error occurred while processing your query', - }; - - return { - isError: true, - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } - } - ); + registerListNamespacesTool(server); + registerNamespaceRouterTool(server); + registerSuggestQueryParamsTool(server); + registerCountTool(server); + registerQueryTool(server); + registerGuidedQueryTool(server); + registerGenerateUrlsTool(server); return server; } diff --git a/src/server/client-context.ts b/src/server/client-context.ts new file mode 100644 index 0000000..9850ba7 --- /dev/null +++ b/src/server/client-context.ts @@ -0,0 +1,15 @@ +import { PineconeClient } from '../pinecone-client.js'; + +// Global Pinecone client (initialized lazily) +let pineconeClient: PineconeClient | null = null; + +export function getPineconeClient(): PineconeClient { + if (!pineconeClient) { + throw new Error('Pinecone client not initialized. Call setPineconeClient first.'); + } + return pineconeClient; +} + +export function setPineconeClient(client: PineconeClient): void { + pineconeClient = client; +} diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts new file mode 100644 index 0000000..879a4ea --- /dev/null +++ b/src/server/metadata-filter.ts @@ -0,0 +1,65 @@ +import { z } from 'zod'; + +// Recursive Zod schema for Pinecone metadata filters +// Supports nested objects with operators like {"timestamp": {"$gte": 123}} +const metadataFilterValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.array(z.string()), + z.array(z.number()), + z.record(z.string(), metadataFilterValueSchema), // Recursive for nested operators + ]) +); + +export const metadataFilterSchema = z.record(z.string(), metadataFilterValueSchema); +const ALLOWED_FILTER_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin']); + +function isPrimitiveFilterValue(value: unknown): boolean { + return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; +} + +function isPrimitiveArray(value: unknown): boolean { + return Array.isArray(value) && value.every((item) => isPrimitiveFilterValue(item)); +} + +function validateMetadataFilterValue(value: unknown, path: string[]): string | null { + if (value === null || value === undefined) { + return `Invalid null/undefined at "${path.join('.')}".`; + } + + if (isPrimitiveFilterValue(value) || isPrimitiveArray(value)) { + return null; + } + + if (typeof value !== 'object' || Array.isArray(value)) { + return `Unsupported filter value at "${path.join('.')}".`; + } + + for (const [key, nestedValue] of Object.entries(value)) { + if (key.startsWith('$')) { + if (!ALLOWED_FILTER_OPERATORS.has(key)) { + return `Unsupported filter operator "${key}" at "${path.join('.')}".`; + } + if ((key === '$in' || key === '$nin') && !isPrimitiveArray(nestedValue)) { + return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; + } + } + + const nestedError = validateMetadataFilterValue(nestedValue, [...path, key]); + if (nestedError) { + return nestedError; + } + } + + return null; +} + +export function validateMetadataFilter(filter: Record): string | null { + for (const [field, value] of Object.entries(filter)) { + const error = validateMetadataFilterValue(value, [field]); + if (error) return error; + } + return null; +} diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts new file mode 100644 index 0000000..bbafe11 --- /dev/null +++ b/src/server/namespace-router.ts @@ -0,0 +1,74 @@ +import type { NamespaceInfo } from './namespaces-cache.js'; + +type RankedNamespace = { + namespace: string; + score: number; + record_count: number; + reasons: string[]; +}; + +function scoreNamespace( + query: string, + namespace: string, + fields: string[] +): { score: number; reasons: string[] } { + const q = query.toLowerCase(); + const name = namespace.toLowerCase(); + const reasons: string[] = []; + let score = 0; + + const keywordHints: Record = { + 'wg21-papers': ['wg21', 'paper', 'proposal', 'document_number', 'p0', 'standard'], + 'github-compiler': ['github', 'issue', 'pr', 'repository', 'compiler', 'bug'], + 'youtube-scripts': ['youtube', 'video', 'script', 'transcript', 'channel'], + 'mailing': ['mailing', 'email', 'thread', 'subject', 'list'], + 'slack-cpplang': ['slack', 'chat', 'message', 'channel', 'thread'], + 'blog-posts': ['blog', 'post', 'article'], + 'cpp-documentation': ['documentation', 'docs', 'reference', 'library'], + }; + + if (q.includes(name.replace(/[^a-z0-9]/g, ' '))) { + score += 3; + reasons.push('query mentions namespace name'); + } + + const hints = keywordHints[name] ?? []; + for (const hint of hints) { + if (q.includes(hint)) { + score += 2; + reasons.push(`keyword match: ${hint}`); + } + } + + for (const field of fields) { + if (q.includes(field.toLowerCase())) { + score += 1; + reasons.push(`field hint: ${field}`); + } + } + + return { score, reasons: Array.from(new Set(reasons)) }; +} + +export function rankNamespacesByQuery( + query: string, + namespaces: NamespaceInfo[], + topN: number +): RankedNamespace[] { + return namespaces + .map((ns) => { + const fields = Object.keys(ns.metadata ?? {}); + const { score, reasons } = scoreNamespace(query.trim(), ns.namespace, fields); + return { + namespace: ns.namespace, + score, + record_count: ns.recordCount, + reasons, + }; + }) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + return a.record_count - b.record_count; + }) + .slice(0, topN); +} diff --git a/src/server/namespaces-cache.ts b/src/server/namespaces-cache.ts new file mode 100644 index 0000000..bd51579 --- /dev/null +++ b/src/server/namespaces-cache.ts @@ -0,0 +1,40 @@ +import { FLOW_CACHE_TTL_MS } from '../constants.js'; +import { getPineconeClient } from './client-context.js'; + +export type NamespaceInfo = { + namespace: string; + recordCount: number; + metadata: Record; +}; + +type CacheEntry = { + data: NamespaceInfo[]; + expiresAt: number; +}; + +let namespacesCache: CacheEntry | null = null; + +export async function getNamespacesWithCache(): Promise<{ + data: NamespaceInfo[]; + cache_hit: boolean; + expires_at: number; +}> { + const now = Date.now(); + if (namespacesCache && now < namespacesCache.expiresAt) { + return { + data: namespacesCache.data, + cache_hit: true, + expires_at: namespacesCache.expiresAt, + }; + } + + const client = getPineconeClient(); + const data = await client.listNamespacesWithMetadata(); + const expiresAt = now + FLOW_CACHE_TTL_MS; + namespacesCache = { data, expiresAt }; + return { data, cache_hit: false, expires_at: expiresAt }; +} + +export function invalidateNamespacesCache(): void { + namespacesCache = null; +} diff --git a/src/server/query-suggestion.ts b/src/server/query-suggestion.ts new file mode 100644 index 0000000..baffbe9 --- /dev/null +++ b/src/server/query-suggestion.ts @@ -0,0 +1,77 @@ +import { COUNT_FIELDS } from '../constants.js'; + +/** Suggested query params from namespace schema and user query. */ +export type SuggestQueryParamsResult = { + suggested_fields: string[]; + use_count_tool: boolean; + recommended_tool: 'count' | 'query_fast' | 'query_detailed'; + explanation: string; + namespace_found: boolean; +}; + +/** + * Suggests which fields to request and whether to use the count tool, + * based on the namespace's available metadata fields (from list_namespaces) and the user's natural language query. + */ +export function suggestQueryParams( + namespaceMetadataFields: Record | null, + userQuery: string +): SuggestQueryParamsResult { + const q = userQuery.toLowerCase().trim(); + const available = namespaceMetadataFields ? Object.keys(namespaceMetadataFields) : []; + const keepOnlyAvailable = (fields: string[]) => fields.filter((f) => available.includes(f)); + + if (!namespaceMetadataFields || available.length === 0) { + return { + suggested_fields: [], + use_count_tool: false, + recommended_tool: 'query_fast', + explanation: 'Namespace not found or has no metadata fields. Call list_namespaces first, then pass a valid namespace.', + namespace_found: false, + }; + } + + // Count intent: "how many", "count", "number of", etc. + if (/\b(how many|count|number of|total number|paper count|documents? count)\b/.test(q)) { + const fields = keepOnlyAvailable([...COUNT_FIELDS]); + return { + suggested_fields: fields.length ? fields : available.slice(0, 5), + use_count_tool: true, + recommended_tool: 'count', + explanation: + 'User asked for a count. Use the count tool for this. If using query instead, use minimal fields (no chunk_text).', + namespace_found: true, + }; + } + + // Content intent: user wants to read/summarize content + if ( + /\b(content|summarize|summarise|what does|excerpt|text|say|details?|full text|body)\b/.test(q) + ) { + const fields = keepOnlyAvailable([ + 'document_number', + 'title', + 'url', + 'author', + 'chunk_text', + ]); + return { + suggested_fields: fields.length ? fields : available, + use_count_tool: false, + recommended_tool: 'query_detailed', + explanation: 'User asked for content or details; include chunk_text for snippets.', + namespace_found: true, + }; + } + + // List/browse intent: titles, links, list (minimal fields, no content) + const listFields = keepOnlyAvailable(['document_number', 'title', 'url', 'author']); + return { + suggested_fields: listFields.length ? listFields : available.slice(0, 5), + use_count_tool: false, + recommended_tool: 'query_fast', + explanation: + 'User asked for a list or browse; use minimal fields (no chunk_text) for smaller payload and cost.', + namespace_found: true, + }; +} diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts new file mode 100644 index 0000000..d80a19c --- /dev/null +++ b/src/server/suggestion-flow.ts @@ -0,0 +1,49 @@ +import { FLOW_CACHE_TTL_MS } from '../constants.js'; + +type FlowState = { + updatedAt: number; + recommended_tool: 'count' | 'query_fast' | 'query_detailed'; + suggested_fields: string[]; + user_query: string; +}; + +const stateByNamespace = new Map(); + +export function markSuggested( + namespace: string, + state: Omit +): void { + stateByNamespace.set(namespace, { + ...state, + updatedAt: Date.now(), + }); +} + +export function requireSuggested(namespace: string): { + ok: true; + flow: FlowState; +} | { + ok: false; + message: string; +} { + const state = stateByNamespace.get(namespace); + if (!state) { + return { + ok: false, + message: + 'Flow requires suggest_query_params first. Call suggest_query_params with namespace and user_query before query/count tools.', + }; + } + + const now = Date.now(); + if (now - state.updatedAt > FLOW_CACHE_TTL_MS) { + stateByNamespace.delete(namespace); + return { + ok: false, + message: + 'Previous suggest_query_params context expired (30 minutes). Call suggest_query_params again before query/count tools.', + }; + } + + return { ok: true, flow: state }; +} diff --git a/src/server/tool-response.ts b/src/server/tool-response.ts new file mode 100644 index 0000000..1ecfec7 --- /dev/null +++ b/src/server/tool-response.ts @@ -0,0 +1,27 @@ +type TextPayload = { + content: Array<{ type: 'text'; text: string }>; + isError?: boolean; +}; + +export function jsonResponse(payload: unknown): TextPayload { + return { + content: [ + { + type: 'text', + text: JSON.stringify(payload, null, 2), + }, + ], + }; +} + +export function jsonErrorResponse(payload: unknown): TextPayload { + return { + isError: true, + content: [ + { + type: 'text', + text: JSON.stringify(payload, null, 2), + }, + ], + }; +} diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts new file mode 100644 index 0000000..f05736b --- /dev/null +++ b/src/server/tools/count-tool.ts @@ -0,0 +1,80 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getPineconeClient } from '../client-context.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { requireSuggested } from '../suggestion-flow.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +const COUNT_RESPONSE_STATUS = 'success' as const; +type CountResponse = + | { status: typeof COUNT_RESPONSE_STATUS; count: number; truncated: boolean; namespace: string; metadata_filter?: Record } + | { status: 'error'; message: string }; + +export function registerCountTool(server: McpServer): void { + server.registerTool( + 'count', + { + description: + 'Get the count of unique documents matching a metadata filter and semantic query. ' + + 'Use when the user asks for a count (e.g. "how many papers by Lakos?", "Lakos\'s paper count"). ' + + 'Uses semantic (dense) search only and requests only document identifiers (no content) for performance. ' + + 'Returns unique document count (deduped by document_number/url/doc_id) up to 10,000; truncated=true if at least that many. ' + + 'Mandatory flow: call suggest_query_params first, then count. ' + + 'Use list_namespaces to discover namespace and metadata fields. ' + + 'For count-by-metadata only, use a broad query_text (e.g. "paper" or "document"). ' + + 'Same metadata_filter operators as query: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin.', + inputSchema: { + namespace: z.string().describe('Namespace to count in. Use list_namespaces to discover namespaces.'), + query_text: z + .string() + .describe('Search query text. Use a broad term (e.g. "paper", "document") when counting by metadata only.'), + metadata_filter: metadataFilterSchema + .optional() + .describe( + 'Optional metadata filter. Use exact field names from list_namespaces. E.g. {"author": {"$in": ["John Lakos"]}} for author count.' + ), + }, + }, + async (params) => { + try { + const { namespace, query_text, metadata_filter } = params; + if (!query_text || !query_text.trim()) { + const response: CountResponse = { status: 'error', message: 'query_text cannot be empty' }; + return jsonErrorResponse(response); + } + if (metadata_filter) { + const err = validateMetadataFilter(metadata_filter); + if (err) { + return jsonErrorResponse({ status: 'error', message: err }); + } + } + const flowCheck = requireSuggested(namespace); + if (!flowCheck.ok) { + return jsonErrorResponse({ status: 'error', message: flowCheck.message }); + } + const client = getPineconeClient(); + const { count, truncated } = await client.count({ + query: query_text.trim(), + namespace, + metadataFilter: metadata_filter, + }); + const response: CountResponse = { + status: COUNT_RESPONSE_STATUS, + count, + truncated, + namespace, + metadata_filter: metadata_filter, + }; + return jsonResponse(response); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error('Error in count tool:', error); + const response: CountResponse = { + status: 'error', + message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to get count', + }; + return jsonErrorResponse(response); + } + } + ); +} diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts new file mode 100644 index 0000000..6e71f14 --- /dev/null +++ b/src/server/tools/generate-urls-tool.ts @@ -0,0 +1,64 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { generateUrlForNamespace } from '../url-generation.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +function extractMetadata(record: Record): Record { + const nested = record.metadata; + if (nested && typeof nested === 'object' && !Array.isArray(nested)) { + return nested as Record; + } + return record; +} + +export function registerGenerateUrlsTool(server: McpServer): void { + server.registerTool( + 'generate_urls', + { + description: + 'Generate URLs for retrieved results when metadata does not include url and URL is needed. ' + + 'Supported namespaces: mailing and slack-Cpplang. For mailing, URL is generated from doc_id or thread_id. ' + + 'For slack-Cpplang, source is preferred when present; otherwise URL is generated from team_id, channel_id, and doc_id.', + inputSchema: { + namespace: z + .string() + .describe('Target namespace. URL generation currently supports mailing and slack-Cpplang.'), + records: z + .array(z.record(z.string(), z.unknown())) + .describe( + 'Array of records from retrieval results. Each item may be either metadata itself or an object containing a metadata field.' + ), + }, + }, + async (params) => { + try { + const { namespace, records } = params; + const results = records.map((record, index) => { + const metadata = extractMetadata(record); + const generated = generateUrlForNamespace(namespace, metadata); + return { + index, + url: generated.url, + method: generated.method, + reason: generated.reason ?? null, + metadata, + }; + }); + + return jsonResponse({ + status: 'success', + namespace, + count: results.length, + results, + }); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error('Error in generate_urls tool:', error); + return jsonErrorResponse({ + status: 'error', + message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to generate URLs', + }); + } + } + ); +} diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts new file mode 100644 index 0000000..0eb6f27 --- /dev/null +++ b/src/server/tools/guided-query-tool.ts @@ -0,0 +1,221 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { FAST_QUERY_FIELDS, MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; +import type { PineconeMetadataValue, QueryResponse, SearchResult } from '../../types.js'; +import { getPineconeClient } from '../client-context.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { rankNamespacesByQuery } from '../namespace-router.js'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { suggestQueryParams } from '../query-suggestion.js'; +import { markSuggested } from '../suggestion-flow.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; +import { generateUrlForNamespace } from '../url-generation.js'; + +type GuidedToolName = 'count' | 'query_fast' | 'query_detailed'; + +function formatQueryResponse( + queryText: string, + namespace: string, + metadataFilter: Record | undefined, + fields: string[] | undefined, + mode: 'query_fast' | 'query_detailed', + results: SearchResult[], + enrichUrls: boolean +): QueryResponse { + const formattedResults = results.map((doc) => { + const metadata = { ...doc.metadata } as Record; + if (enrichUrls) { + const generated = generateUrlForNamespace(namespace, metadata); + if (generated.url && typeof metadata.url !== 'string') { + metadata.url = generated.url; + } + } + const docNum = metadata.document_number; + const filename = metadata.filename; + const paper_number = + (typeof docNum === 'string' ? docNum : null) ?? + (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? + null; + return { + paper_number, + title: String(metadata.title ?? ''), + author: String(metadata.author ?? ''), + url: String(metadata.url ?? ''), + content: doc.content.substring(0, 2000), + score: Math.round(doc.score * 10000) / 10000, + reranked: doc.reranked, + metadata, + }; + }); + + return { + status: 'success', + mode, + query: queryText, + namespace, + metadata_filter: metadataFilter, + result_count: formattedResults.length, + ...(fields?.length ? { fields } : {}), + results: formattedResults, + }; +} + +export function registerGuidedQueryTool(server: McpServer): void { + server.registerTool( + 'guided_query', + { + description: + 'Single orchestrator that runs routing + suggestion + execution in one call. ' + + 'Flow: optional namespace_router logic -> suggest_query_params logic -> executes count/query_fast/query_detailed. ' + + 'Returns decision_trace so behavior stays transparent and debuggable.', + inputSchema: { + user_query: z.string().describe('User question or intent.'), + namespace: z + .string() + .optional() + .describe('Optional explicit namespace. If omitted, namespace_router logic will choose one.'), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to constrain results.'), + top_k: z + .number() + .int() + .min(MIN_TOP_K) + .max(MAX_TOP_K) + .default(10) + .describe('Result count for query_fast/query_detailed paths (1-100).'), + preferred_tool: z + .enum(['auto', 'count', 'query_fast', 'query_detailed']) + .default('auto') + .describe('Optional override. Use auto to follow suggestion logic.'), + enrich_urls: z + .boolean() + .default(true) + .describe( + 'If true, enrich result URLs for mailing/slack-Cpplang when metadata.url is missing.' + ), + }, + }, + async (params) => { + try { + const { + user_query, + namespace: inputNamespace, + metadata_filter, + top_k = 10, + preferred_tool = 'auto', + enrich_urls = true, + } = params; + + if (!user_query?.trim()) { + return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); + } + + if (metadata_filter) { + const err = validateMetadataFilter(metadata_filter); + if (err) { + return jsonErrorResponse({ status: 'error', message: err }); + } + } + + const queryText = user_query.trim(); + const { data: namespaces, cache_hit } = await getNamespacesWithCache(); + const ranked = rankNamespacesByQuery(queryText, namespaces, 3); + + const namespace = inputNamespace ?? ranked[0]?.namespace; + if (!namespace) { + return jsonErrorResponse({ + status: 'error', + message: 'No namespace available. Please run list_namespaces and verify index data.', + }); + } + + const ns = namespaces.find((n) => n.namespace === namespace); + const suggestion = suggestQueryParams(ns?.metadata ?? null, queryText); + if (!suggestion.namespace_found) { + return jsonErrorResponse({ + status: 'error', + message: `Namespace "${namespace}" not found in cached namespaces. Call list_namespaces and retry.`, + }); + } + + const selectedTool: GuidedToolName = + preferred_tool === 'auto' ? suggestion.recommended_tool : preferred_tool; + markSuggested(namespace, { + recommended_tool: selectedTool, + suggested_fields: suggestion.suggested_fields, + user_query: queryText, + }); + + const client = getPineconeClient(); + const decision_trace = { + cache_hit, + input_namespace: inputNamespace ?? null, + routed_namespace: ranked[0]?.namespace ?? null, + selected_namespace: namespace, + ranked_namespaces: ranked, + suggested_fields: suggestion.suggested_fields, + suggested_tool: suggestion.recommended_tool, + selected_tool: selectedTool, + explanation: suggestion.explanation, + enrich_urls, + }; + + if (selectedTool === 'count') { + const { count, truncated } = await client.count({ + query: queryText, + namespace, + metadataFilter: metadata_filter, + }); + return jsonResponse({ + status: 'success', + decision_trace, + result: { + tool: 'count', + namespace, + query: queryText, + metadata_filter, + count, + truncated, + }, + }); + } + + const isFast = selectedTool === 'query_fast'; + const fields = + suggestion.suggested_fields.length > 0 + ? suggestion.suggested_fields + : [...FAST_QUERY_FIELDS]; + const queryResults = await client.query({ + query: queryText, + namespace, + topK: top_k, + metadataFilter: metadata_filter, + useReranking: !isFast, + fields, + }); + const result = formatQueryResponse( + queryText, + namespace, + metadata_filter, + fields, + isFast ? 'query_fast' : 'query_detailed', + queryResults, + enrich_urls + ); + return jsonResponse({ + status: 'success', + decision_trace, + result, + }); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error('Error in guided_query tool:', error); + return jsonErrorResponse({ + status: 'error', + message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to execute guided query', + }); + } + } + ); +} diff --git a/src/server/tools/list-namespaces-tool.ts b/src/server/tools/list-namespaces-tool.ts new file mode 100644 index 0000000..65ce619 --- /dev/null +++ b/src/server/tools/list-namespaces-tool.ts @@ -0,0 +1,48 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +export function registerListNamespacesTool(server: McpServer): void { + server.registerTool( + 'list_namespaces', + { + description: + 'List all available namespaces in the Pinecone index with their metadata fields and record counts. ' + + 'Returns detailed information about each namespace including available metadata fields that can be used for filtering in queries. ' + + 'Use this tool first to discover which namespaces exist and what metadata fields are available for filtering. ' + + 'Results are cached in-memory for 30 minutes for better performance.', + inputSchema: {}, + }, + async () => { + try { + const { data: namespacesInfo, cache_hit, expires_at } = await getNamespacesWithCache(); + const now = Date.now(); + const ttlSeconds = Math.max(0, Math.floor((expires_at - now) / 1000)); + + const response = { + status: 'success', + cache_hit, + cache_ttl_seconds: ttlSeconds, + count: namespacesInfo.length, + namespaces: namespacesInfo.map((ns) => ({ + name: ns.namespace, + record_count: ns.recordCount, + metadata_fields: ns.metadata, + })), + }; + + return jsonResponse(response); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error('Error listing namespaces:', error); + + const response = { + status: 'error', + message: process.env.LOG_LEVEL === 'DEBUG' ? errorMessage : 'Failed to list namespaces', + }; + + return jsonErrorResponse(response); + } + } + ); +} diff --git a/src/server/tools/namespace-router-tool.ts b/src/server/tools/namespace-router-tool.ts new file mode 100644 index 0000000..36e2941 --- /dev/null +++ b/src/server/tools/namespace-router-tool.ts @@ -0,0 +1,46 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { rankNamespacesByQuery } from '../namespace-router.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +export function registerNamespaceRouterTool(server: McpServer): void { + server.registerTool( + 'namespace_router', + { + description: + 'Suggest likely namespace(s) for a user query using namespace names, metadata fields, and keyword heuristics. ' + + 'Use before suggest_query_params when namespace is unclear.', + inputSchema: { + user_query: z.string().describe('User question/intent used to infer relevant namespace(s).'), + top_n: z.number().int().min(1).max(5).default(3).describe('Maximum number of suggested namespaces (1-5).'), + }, + }, + async (params) => { + try { + const { user_query, top_n = 3 } = params; + if (!user_query?.trim()) { + return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); + } + const { data, cache_hit } = await getNamespacesWithCache(); + const ranked = rankNamespacesByQuery(user_query.trim(), data, top_n); + + const response = { + status: 'success' as const, + cache_hit, + user_query: user_query.trim(), + suggestions: ranked, + recommended_namespace: ranked[0]?.namespace ?? null, + }; + return jsonResponse(response); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error('Error in namespace_router tool:', error); + return jsonErrorResponse({ + status: 'error', + message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to route namespace', + }); + } + } + ); +} diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts new file mode 100644 index 0000000..d1823a7 --- /dev/null +++ b/src/server/tools/query-tool.ts @@ -0,0 +1,205 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { FAST_QUERY_FIELDS, MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; +import type { QueryResponse } from '../../types.js'; +import { getPineconeClient } from '../client-context.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { requireSuggested } from '../suggestion-flow.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +type QueryMode = 'query' | 'query_fast' | 'query_detailed'; + +type QueryExecParams = { + query_text: string; + namespace: string; + top_k: number; + use_reranking: boolean; + metadata_filter?: Record; + fields?: string[]; + mode: QueryMode; +}; + +async function executeQuery(params: QueryExecParams) { + try { + const { + query_text, + namespace, + top_k, + use_reranking, + metadata_filter, + fields, + mode, + } = params; + + if (!query_text || !query_text.trim()) { + const response: QueryResponse = { + status: 'error', + message: 'Query text cannot be empty', + }; + return jsonErrorResponse(response); + } + + if (metadata_filter) { + const filterValidationError = validateMetadataFilter(metadata_filter); + if (filterValidationError) { + const response: QueryResponse = { + status: 'error', + message: filterValidationError, + }; + return jsonErrorResponse(response); + } + } + + const flowCheck = requireSuggested(namespace); + if (!flowCheck.ok) { + return jsonErrorResponse({ status: 'error', message: flowCheck.message }); + } + + if (metadata_filter) { + console.error('Received metadata filter:', JSON.stringify(metadata_filter, null, 2)); + } + + const client = getPineconeClient(); + const results = await client.query({ + query: query_text.trim(), + topK: top_k, + namespace, + useReranking: use_reranking, + metadataFilter: metadata_filter, + fields: fields?.length ? fields : undefined, + }); + + const formattedResults = results.map((doc) => { + const docNum = doc.metadata.document_number; + const filename = doc.metadata.filename; + const paper_number = + (typeof docNum === 'string' ? docNum : null) ?? + (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? + null; + return { + paper_number, + title: String(doc.metadata.title ?? ''), + author: String(doc.metadata.author ?? ''), + url: String(doc.metadata.url ?? ''), + content: doc.content.substring(0, 2000), + score: Math.round(doc.score * 10000) / 10000, + reranked: doc.reranked, + metadata: doc.metadata, + }; + }); + + const response: QueryResponse = { + status: 'success', + mode, + query: query_text, + namespace, + metadata_filter: metadata_filter, + result_count: formattedResults.length, + results: formattedResults, + ...(fields?.length ? { fields } : {}), + }; + return jsonResponse(response); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + console.error('Error executing query:', error); + const response: QueryResponse = { + status: 'error', + message: + process.env.LOG_LEVEL === 'DEBUG' + ? errorMessage + : 'An error occurred while processing your query', + }; + return jsonErrorResponse(response); + } +} + +const baseSchema = { + query_text: z.string().describe('Search query text. Be specific for better results.'), + namespace: z + .string() + .describe( + 'Namespace to search within. Use list_namespaces/namespace_router first, then suggest_query_params before querying.' + ), + top_k: z + .number() + .int() + .min(MIN_TOP_K) + .max(MAX_TOP_K) + .default(10) + .describe('Number of results to return (1-100). Default: 10'), + metadata_filter: metadataFilterSchema.optional().describe('Optional metadata filter to narrow down search results.'), + fields: z + .array(z.string()) + .optional() + .describe( + 'Optional field names to return from Pinecone. Use suggest_query_params suggested_fields for better performance.' + ), +}; + +export function registerQueryTool(server: McpServer): void { + server.registerTool( + 'query', + { + description: + 'Full query tool with optional reranking. Mandatory flow: call suggest_query_params first. ' + + 'For lighter retrieval use query_fast; for content-heavy retrieval use query_detailed.', + inputSchema: { + ...baseSchema, + use_reranking: z + .boolean() + .default(true) + .describe('Whether to use semantic reranking for better relevance. Slower but more accurate.'), + }, + }, + async (params) => + executeQuery({ + ...params, + top_k: params.top_k ?? 10, + use_reranking: params.use_reranking ?? true, + mode: 'query', + }) + ); + + server.registerTool( + 'query_fast', + { + description: + 'Fast query preset. Mandatory flow: call suggest_query_params first. ' + + 'Defaults to no reranking and lightweight fields for lower latency/cost.', + inputSchema: { + ...baseSchema, + }, + }, + async (params) => + executeQuery({ + ...params, + top_k: params.top_k ?? 10, + use_reranking: false, + fields: params.fields?.length ? params.fields : [...FAST_QUERY_FIELDS], + mode: 'query_fast', + }) + ); + + server.registerTool( + 'query_detailed', + { + description: + 'Detailed query preset. Mandatory flow: call suggest_query_params first. ' + + 'Designed for reading/summarization workflows with content snippets.', + inputSchema: { + ...baseSchema, + use_reranking: z + .boolean() + .default(true) + .describe('Whether to use semantic reranking for better precision (default true).'), + }, + }, + async (params) => + executeQuery({ + ...params, + top_k: params.top_k ?? 10, + use_reranking: params.use_reranking ?? true, + mode: 'query_detailed', + }) + ); +} diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts new file mode 100644 index 0000000..3f46359 --- /dev/null +++ b/src/server/tools/suggest-query-params-tool.ts @@ -0,0 +1,63 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { suggestQueryParams } from '../query-suggestion.js'; +import { markSuggested } from '../suggestion-flow.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +export function registerSuggestQueryParamsTool(server: McpServer): void { + server.registerTool( + 'suggest_query_params', + { + description: + 'Suggest which fields to request and whether to use the count tool, based on the namespace schema (from list_namespaces) and the user\'s natural language query. ' + + 'Call list_namespaces first to get available namespaces and metadata fields. Then call this tool with the target namespace and the user query; ' + + 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count/query_fast/query_detailed), and an explanation. ' + + 'This step is mandatory before query/count tools; use the returned suggested_fields in query tools to reduce payload and cost.', + inputSchema: { + namespace: z + .string() + .describe( + 'Namespace to query. Must match a name from list_namespaces so the tool can look up available metadata fields.' + ), + user_query: z + .string() + .describe( + 'The user\'s natural language question or intent (e.g. "list papers by Lakos with titles and links", "how many papers by Wong?", "what do the contracts papers say?").' + ), + }, + }, + async (params) => { + try { + const { namespace, user_query } = params; + if (!user_query?.trim()) { + return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); + } + const { data: namespacesInfo, cache_hit } = await getNamespacesWithCache(); + const ns = namespacesInfo.find((n) => n.namespace === namespace); + const metadataFields = ns?.metadata ?? null; + const result = suggestQueryParams(metadataFields, user_query.trim()); + if (result.namespace_found) { + markSuggested(namespace, { + recommended_tool: result.recommended_tool, + suggested_fields: result.suggested_fields, + user_query: user_query.trim(), + }); + } + const response = { + status: 'success' as const, + cache_hit, + ...result, + }; + return jsonResponse(response); + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.error('Error in suggest_query_params tool:', error); + return jsonErrorResponse({ + status: 'error', + message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to suggest query params', + }); + } + } + ); +} diff --git a/src/server/url-generation.test.ts b/src/server/url-generation.test.ts new file mode 100644 index 0000000..b6a6edf --- /dev/null +++ b/src/server/url-generation.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { generateUrlForNamespace } from './url-generation.js'; + +describe('generateUrlForNamespace', () => { + it('uses existing metadata.url when present', () => { + const r = generateUrlForNamespace('mailing', { + url: 'https://example.com/custom', + doc_id: 'ignored', + }); + expect(r.url).toBe('https://example.com/custom'); + expect(r.method).toBe('metadata.url'); + }); + + it('generates mailing URL from doc_id', () => { + const r = generateUrlForNamespace('mailing', { + doc_id: 'boost-announce@lists.boost.org/message/O5VYCDZADVDHK5Z5LAYJBHMDOAFQL7P6', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-announce@lists.boost.org/message/O5VYCDZADVDHK5Z5LAYJBHMDOAFQL7P6/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('generates mailing URL from thread_id when doc_id missing', () => { + const r = generateUrlForNamespace('mailing', { + thread_id: 'boost@lists.boost.org/thread/ABC123', + }); + expect(r.url).toBe('https://lists.boost.org/archives/list/boost@lists.boost.org/thread/ABC123/'); + expect(r.method).toBe('generated.mailing'); + }); + + it('uses slack source when available', () => { + const r = generateUrlForNamespace('slack-Cpplang', { + source: 'https://app.slack.com/client/T123/C123/p123', + team_id: 'T999', + channel_id: 'C999', + doc_id: '1.2', + }); + expect(r.url).toBe('https://app.slack.com/client/T123/C123/p123'); + expect(r.method).toBe('metadata.source'); + }); + + it('generates slack URL from team/channel/doc_id', () => { + const r = generateUrlForNamespace('slack-Cpplang', { + team_id: 'T123456789', + channel_id: 'C123456', + doc_id: '1234567.890', + }); + expect(r.url).toBe('https://app.slack.com/T123456789/C123456/p1234567890'); + expect(r.method).toBe('generated.slack'); + }); + + it('returns unavailable for unsupported namespace', () => { + const r = generateUrlForNamespace('wg21-papers', { doc_id: 'x' }); + expect(r.url).toBeNull(); + expect(r.method).toBe('unavailable'); + }); +}); diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts new file mode 100644 index 0000000..c250c05 --- /dev/null +++ b/src/server/url-generation.ts @@ -0,0 +1,64 @@ +export type UrlGenerationResult = { + url: string | null; + method: 'metadata.url' | 'metadata.source' | 'generated.mailing' | 'generated.slack' | 'unavailable'; + reason?: string; +}; + +function asString(value: unknown): string | null { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; +} + +export function generateUrlForNamespace( + namespace: string, + metadata: Record +): UrlGenerationResult { + const existingUrl = asString(metadata.url); + if (existingUrl) { + return { url: existingUrl, method: 'metadata.url' }; + } + + if (namespace === 'mailing') { + const docIdOrThread = asString(metadata.doc_id) ?? asString(metadata.thread_id); + if (!docIdOrThread) { + return { + url: null, + method: 'unavailable', + reason: 'mailing requires doc_id or thread_id to generate URL', + }; + } + return { + url: `https://lists.boost.org/archives/list/${docIdOrThread}/`, + method: 'generated.mailing', + }; + } + + if (namespace === 'slack-Cpplang') { + const source = asString(metadata.source); + if (source) { + return { url: source, method: 'metadata.source' }; + } + + const teamId = asString(metadata.team_id); + const channelId = asString(metadata.channel_id); + const docId = asString(metadata.doc_id); + if (!teamId || !channelId || !docId) { + return { + url: null, + method: 'unavailable', + reason: 'slack-Cpplang requires team_id, channel_id, and doc_id (or source)', + }; + } + + const messageId = docId.replace(/\./g, ''); + return { + url: `https://app.slack.com/${teamId}/${channelId}/p${messageId}`, + method: 'generated.slack', + }; + } + + return { + url: null, + method: 'unavailable', + reason: `URL generation is not supported for namespace "${namespace}"`, + }; +} diff --git a/src/types.ts b/src/types.ts index b29ff84..007ef8e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,6 +2,9 @@ * Types for Pinecone Read-Only MCP */ +/** Pinecone metadata value types: string, number, boolean, or list of strings */ +export type PineconeMetadataValue = string | number | boolean | string[]; + export interface PineconeClientConfig { apiKey: string; indexName?: string; @@ -13,14 +16,14 @@ export interface SearchResult { id: string; content: string; score: number; - metadata: Record; + metadata: Record; reranked: boolean; } export interface PineconeHit { _id: string; _score: number; - fields: Record; + fields: Record; } export interface PineconeSearchResponse { @@ -30,15 +33,30 @@ export interface PineconeSearchResponse { } export interface NamespaceStats { - namespaces?: Record; + namespaces?: Record; } export interface QueryParams { query: string; topK?: number; namespace: string; - metadataFilter?: Record; + metadataFilter?: Record; useReranking?: boolean; + /** If set, only these fields are requested from Pinecone (e.g. ["document_number", "title", "url"]). Omit for all fields. Include "chunk_text" for content. */ + fields?: string[]; +} + +/** Parameters for count-only requests (high top_k, no reranking). */ +export interface CountParams { + query: string; + namespace: string; + metadataFilter?: Record; +} + +/** Result of a count request: unique document count (deduped by doc id/url); truncated when at least COUNT_TOP_K. */ +export interface CountResult { + count: number; + truncated: boolean; } export interface ListNamespacesResponse { @@ -50,10 +68,13 @@ export interface ListNamespacesResponse { export interface QueryResponse { status: 'success' | 'error'; + mode?: 'query' | 'query_fast' | 'query_detailed'; query?: string; namespace?: string; - metadata_filter?: Record; + metadata_filter?: Record; result_count?: number; + /** Present when the query requested specific fields. */ + fields?: string[]; results?: Array<{ paper_number: string | null; title: string; @@ -62,7 +83,31 @@ export interface QueryResponse { content: string; score: number; reranked: boolean; - metadata?: Record; // Include all metadata fields + metadata?: Record; }>; message?: string; } + +/** Internal merged hit shape before rerank (dense + sparse deduped). */ +export interface MergedHit { + _id: string; + _score: number; + chunk_text: string; + metadata: Record; +} + +/** Minimal index interface used for hybrid search (dense/sparse) and namespace discovery. */ +export interface SearchableIndex { + describeIndexStats?(): Promise<{ dimension?: number; namespaces?: Record }>; + search?(opts: { + namespace?: string; + query: Record; + fields?: string[]; + }): Promise<{ result?: { hits?: PineconeHit[] } }>; + namespace?(name: string): SearchableIndex & { + query?(opts: { topK: number; vector: number[]; includeMetadata: boolean }): Promise<{ + matches?: Array<{ metadata?: Record }>; + }>; + }; + searchRecords?(params: { query: Record }): Promise<{ result?: { hits?: PineconeHit[] } }>; +} diff --git a/tsconfig.json b/tsconfig.json index 7e99739..3b67967 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,8 +13,16 @@ "declaration": true, "declarationMap": true, "sourceMap": true, - "typeRoots": ["./node_modules/@types"] + "typeRoots": [ + "./node_modules/@types" + ] }, - "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "**/*.test.ts"] -} + "include": [ + "src/**/*" + ], + "exclude": [ + "node_modules", + "dist", + "**/*.test.ts" + ] +} \ No newline at end of file From 2467cc63f11617c66ba923130d6f6f468c2410a6 Mon Sep 17 00:00:00 2001 From: zho Date: Mon, 16 Feb 2026 09:01:05 +0800 Subject: [PATCH 02/20] #15-optimized code --- src/index.ts | 10 ++- src/pinecone-client.ts | 9 ++- src/server/format-query-result.ts | 75 ++++++++++++++++++ src/server/tool-error.ts | 16 ++++ src/server/tools/count-tool.ts | 6 +- src/server/tools/generate-urls-tool.ts | 6 +- src/server/tools/guided-query-tool.ts | 78 +++++-------------- src/server/tools/list-namespaces-tool.ts | 8 +- src/server/tools/namespace-router-tool.ts | 6 +- src/server/tools/query-tool.ts | 33 ++------ src/server/tools/suggest-query-params-tool.ts | 6 +- 11 files changed, 142 insertions(+), 111 deletions(-) create mode 100644 src/server/format-query-result.ts create mode 100644 src/server/tool-error.ts diff --git a/src/index.ts b/src/index.ts index 1fa42e5..fd9a701 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,8 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js' import { PineconeClient } from './pinecone-client.js'; import { setupServer, setPineconeClient } from './server.js'; import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL } from './constants.js'; +import type { LogLevel } from './config.js'; +import { setLogLevel } from './logger.js'; import * as dotenv from 'dotenv'; // Load environment variables @@ -96,13 +98,17 @@ async function main(): Promise { try { const options = parseArgs(); - // Set log level - const logLevel = + // Set log level (env + logger singleton so tools get correct level) + const rawLevel = options.logLevel || process.env.PINECONE_READ_ONLY_MCP_LOG_LEVEL || process.env.LOG_LEVEL || 'INFO'; + const logLevel = (['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) + ? rawLevel + : 'INFO') as LogLevel; process.env.LOG_LEVEL = logLevel; + setLogLevel(logLevel); // Get API key const apiKey = options.apiKey || process.env.PINECONE_API_KEY; diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 3490b1f..5262e2a 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -7,6 +7,7 @@ */ import { Pinecone } from '@pinecone-database/pinecone'; +import { debug as logDebug, error as logError, info as logInfo } from './logger.js'; import type { PineconeClientConfig, SearchResult, @@ -77,7 +78,7 @@ export class PineconeClient { ); } this.pc = new Pinecone({ apiKey: this.apiKey }); - console.error('Pinecone client initialized'); + logInfo('Pinecone client initialized'); } return this.pc; } @@ -100,7 +101,7 @@ export class PineconeClient { this.sparseIndex = sparse; this.initialized = true; - console.error(`Connected to indexes: ${denseName} and ${sparseName}`); + logInfo(`Connected to indexes: ${denseName} and ${sparseName}`); return { denseIndex: dense, sparseIndex: sparse }; } @@ -215,7 +216,7 @@ export class PineconeClient { if (metadataFilter !== undefined) { queryPayload.filter = metadataFilter; if (!options?.fields) { - console.error('Applying metadata filter:', JSON.stringify(metadataFilter)); + logDebug('Applying metadata filter', metadataFilter); } } @@ -335,7 +336,7 @@ export class PineconeClient { } return reranked; } catch (error) { - console.error('Error reranking results:', error); + logError('Error reranking results', error); // Fall back to returning unreranked results return results.slice(0, topN).map((result) => ({ id: result._id || '', diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts new file mode 100644 index 0000000..19486d1 --- /dev/null +++ b/src/server/format-query-result.ts @@ -0,0 +1,75 @@ +/** + * Shared formatting of Pinecone SearchResult into QueryResponse result rows. + * Used by query tool and guided_query to avoid duplicated paper_number/title/author/url logic. + */ + +import type { PineconeMetadataValue, SearchResult } from '../types.js'; +import { generateUrlForNamespace } from './url-generation.js'; + +const DEFAULT_CONTENT_MAX_LENGTH = 2000; + +export interface QueryResultRow { + paper_number: string | null; + title: string; + author: string; + url: string; + content: string; + score: number; + reranked: boolean; + metadata?: Record; +} + +/** + * Format a single search result into a QueryResponse result row. + * Optionally enrich url from namespace (mailing/slack-Cpplang) when metadata.url is missing. + */ +export function formatSearchResultAsRow( + doc: SearchResult, + options?: { + namespace?: string; + enrichUrls?: boolean; + contentMaxLength?: number; + } +): QueryResultRow { + const contentMaxLength = options?.contentMaxLength ?? DEFAULT_CONTENT_MAX_LENGTH; + const metadata = { ...doc.metadata } as Record; + + if (options?.enrichUrls && options?.namespace) { + const generated = generateUrlForNamespace(options.namespace, metadata); + if (generated.url && typeof metadata.url !== 'string') { + metadata.url = generated.url; + } + } + + const docNum = metadata.document_number; + const filename = metadata.filename; + const paper_number = + (typeof docNum === 'string' ? docNum : null) ?? + (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? + null; + + return { + paper_number, + title: String(metadata.title ?? ''), + author: String(metadata.author ?? ''), + url: String(metadata.url ?? ''), + content: doc.content.substring(0, contentMaxLength), + score: Math.round(doc.score * 10000) / 10000, + reranked: doc.reranked, + metadata, + }; +} + +/** + * Format an array of search results into QueryResponse result rows. + */ +export function formatQueryResultRows( + results: SearchResult[], + options?: { + namespace?: string; + enrichUrls?: boolean; + contentMaxLength?: number; + } +): QueryResultRow[] { + return results.map((doc) => formatSearchResultAsRow(doc, options)); +} diff --git a/src/server/tool-error.ts b/src/server/tool-error.ts new file mode 100644 index 0000000..ec2274a --- /dev/null +++ b/src/server/tool-error.ts @@ -0,0 +1,16 @@ +/** + * Shared error handling for MCP tools: consistent logging and user-facing messages. + */ + +import { getLogLevel, error as logError } from '../logger.js'; + +/** User-facing error message: detailed in DEBUG, generic otherwise. */ +export function getToolErrorMessage(error: unknown, fallbackMessage: string): string { + const msg = error instanceof Error ? error.message : String(error); + return getLogLevel() === 'DEBUG' ? msg : fallbackMessage; +} + +/** Log tool failure to stderr via the level-based logger. */ +export function logToolError(toolName: string, error: unknown): void { + logError(`Error in ${toolName} tool`, error); +} diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts index f05736b..af26624 100644 --- a/src/server/tools/count-tool.ts +++ b/src/server/tools/count-tool.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { getPineconeClient } from '../client-context.js'; import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; import { requireSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; const COUNT_RESPONSE_STATUS = 'success' as const; @@ -67,11 +68,10 @@ export function registerCountTool(server: McpServer): void { }; return jsonResponse(response); } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error('Error in count tool:', error); + logToolError('count', error); const response: CountResponse = { status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to get count', + message: getToolErrorMessage(error, 'Failed to get count'), }; return jsonErrorResponse(response); } diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts index 6e71f14..cb41c75 100644 --- a/src/server/tools/generate-urls-tool.ts +++ b/src/server/tools/generate-urls-tool.ts @@ -1,6 +1,7 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; import { generateUrlForNamespace } from '../url-generation.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; function extractMetadata(record: Record): Record { @@ -52,11 +53,10 @@ export function registerGenerateUrlsTool(server: McpServer): void { results, }); } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error('Error in generate_urls tool:', error); + logToolError('generate_urls', error); return jsonErrorResponse({ status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to generate URLs', + message: getToolErrorMessage(error, 'Failed to generate URLs'), }); } } diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts index 0eb6f27..8a9ba37 100644 --- a/src/server/tools/guided-query-tool.ts +++ b/src/server/tools/guided-query-tool.ts @@ -1,65 +1,19 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; import { FAST_QUERY_FIELDS, MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; -import type { PineconeMetadataValue, QueryResponse, SearchResult } from '../../types.js'; +import type { QueryResponse } from '../../types.js'; import { getPineconeClient } from '../client-context.js'; +import { formatQueryResultRows } from '../format-query-result.js'; import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; import { rankNamespacesByQuery } from '../namespace-router.js'; import { getNamespacesWithCache } from '../namespaces-cache.js'; import { suggestQueryParams } from '../query-suggestion.js'; import { markSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; -import { generateUrlForNamespace } from '../url-generation.js'; type GuidedToolName = 'count' | 'query_fast' | 'query_detailed'; -function formatQueryResponse( - queryText: string, - namespace: string, - metadataFilter: Record | undefined, - fields: string[] | undefined, - mode: 'query_fast' | 'query_detailed', - results: SearchResult[], - enrichUrls: boolean -): QueryResponse { - const formattedResults = results.map((doc) => { - const metadata = { ...doc.metadata } as Record; - if (enrichUrls) { - const generated = generateUrlForNamespace(namespace, metadata); - if (generated.url && typeof metadata.url !== 'string') { - metadata.url = generated.url; - } - } - const docNum = metadata.document_number; - const filename = metadata.filename; - const paper_number = - (typeof docNum === 'string' ? docNum : null) ?? - (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? - null; - return { - paper_number, - title: String(metadata.title ?? ''), - author: String(metadata.author ?? ''), - url: String(metadata.url ?? ''), - content: doc.content.substring(0, 2000), - score: Math.round(doc.score * 10000) / 10000, - reranked: doc.reranked, - metadata, - }; - }); - - return { - status: 'success', - mode, - query: queryText, - namespace, - metadata_filter: metadataFilter, - result_count: formattedResults.length, - ...(fields?.length ? { fields } : {}), - results: formattedResults, - }; -} - export function registerGuidedQueryTool(server: McpServer): void { server.registerTool( 'guided_query', @@ -194,26 +148,30 @@ export function registerGuidedQueryTool(server: McpServer): void { useReranking: !isFast, fields, }); - const result = formatQueryResponse( - queryText, + const formattedResults = formatQueryResultRows(queryResults, { namespace, - metadata_filter, - fields, - isFast ? 'query_fast' : 'query_detailed', - queryResults, - enrich_urls - ); + enrichUrls: enrich_urls, + }); + const result: QueryResponse = { + status: 'success', + mode: isFast ? 'query_fast' : 'query_detailed', + query: queryText, + namespace, + metadata_filter: metadata_filter, + result_count: formattedResults.length, + ...(fields.length > 0 ? { fields } : {}), + results: formattedResults, + }; return jsonResponse({ status: 'success', decision_trace, result, }); } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error('Error in guided_query tool:', error); + logToolError('guided_query', error); return jsonErrorResponse({ status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to execute guided query', + message: getToolErrorMessage(error, 'Failed to execute guided query'), }); } } diff --git a/src/server/tools/list-namespaces-tool.ts b/src/server/tools/list-namespaces-tool.ts index 65ce619..92487fb 100644 --- a/src/server/tools/list-namespaces-tool.ts +++ b/src/server/tools/list-namespaces-tool.ts @@ -1,5 +1,6 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; export function registerListNamespacesTool(server: McpServer): void { @@ -33,14 +34,11 @@ export function registerListNamespacesTool(server: McpServer): void { return jsonResponse(response); } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('Error listing namespaces:', error); - + logToolError('list_namespaces', error); const response = { status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? errorMessage : 'Failed to list namespaces', + message: getToolErrorMessage(error, 'Failed to list namespaces'), }; - return jsonErrorResponse(response); } } diff --git a/src/server/tools/namespace-router-tool.ts b/src/server/tools/namespace-router-tool.ts index 36e2941..65e17e5 100644 --- a/src/server/tools/namespace-router-tool.ts +++ b/src/server/tools/namespace-router-tool.ts @@ -2,6 +2,7 @@ import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; import { getNamespacesWithCache } from '../namespaces-cache.js'; import { rankNamespacesByQuery } from '../namespace-router.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; export function registerNamespaceRouterTool(server: McpServer): void { @@ -34,11 +35,10 @@ export function registerNamespaceRouterTool(server: McpServer): void { }; return jsonResponse(response); } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error('Error in namespace_router tool:', error); + logToolError('namespace_router', error); return jsonErrorResponse({ status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to route namespace', + message: getToolErrorMessage(error, 'Failed to route namespace'), }); } } diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts index d1823a7..c54a4ff 100644 --- a/src/server/tools/query-tool.ts +++ b/src/server/tools/query-tool.ts @@ -3,8 +3,10 @@ import { z } from 'zod'; import { FAST_QUERY_FIELDS, MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; import type { QueryResponse } from '../../types.js'; import { getPineconeClient } from '../client-context.js'; +import { formatQueryResultRows } from '../format-query-result.js'; import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; import { requireSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; type QueryMode = 'query' | 'query_fast' | 'query_detailed'; @@ -55,10 +57,6 @@ async function executeQuery(params: QueryExecParams) { return jsonErrorResponse({ status: 'error', message: flowCheck.message }); } - if (metadata_filter) { - console.error('Received metadata filter:', JSON.stringify(metadata_filter, null, 2)); - } - const client = getPineconeClient(); const results = await client.query({ query: query_text.trim(), @@ -69,24 +67,7 @@ async function executeQuery(params: QueryExecParams) { fields: fields?.length ? fields : undefined, }); - const formattedResults = results.map((doc) => { - const docNum = doc.metadata.document_number; - const filename = doc.metadata.filename; - const paper_number = - (typeof docNum === 'string' ? docNum : null) ?? - (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? - null; - return { - paper_number, - title: String(doc.metadata.title ?? ''), - author: String(doc.metadata.author ?? ''), - url: String(doc.metadata.url ?? ''), - content: doc.content.substring(0, 2000), - score: Math.round(doc.score * 10000) / 10000, - reranked: doc.reranked, - metadata: doc.metadata, - }; - }); + const formattedResults = formatQueryResultRows(results); const response: QueryResponse = { status: 'success', @@ -100,14 +81,10 @@ async function executeQuery(params: QueryExecParams) { }; return jsonResponse(response); } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('Error executing query:', error); + logToolError('query', error); const response: QueryResponse = { status: 'error', - message: - process.env.LOG_LEVEL === 'DEBUG' - ? errorMessage - : 'An error occurred while processing your query', + message: getToolErrorMessage(error, 'An error occurred while processing your query'), }; return jsonErrorResponse(response); } diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts index 3f46359..f40d8cf 100644 --- a/src/server/tools/suggest-query-params-tool.ts +++ b/src/server/tools/suggest-query-params-tool.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { getNamespacesWithCache } from '../namespaces-cache.js'; import { suggestQueryParams } from '../query-suggestion.js'; import { markSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; export function registerSuggestQueryParamsTool(server: McpServer): void { @@ -51,11 +52,10 @@ export function registerSuggestQueryParamsTool(server: McpServer): void { }; return jsonResponse(response); } catch (error) { - const msg = error instanceof Error ? error.message : String(error); - console.error('Error in suggest_query_params tool:', error); + logToolError('suggest_query_params', error); return jsonErrorResponse({ status: 'error', - message: process.env.LOG_LEVEL === 'DEBUG' ? msg : 'Failed to suggest query params', + message: getToolErrorMessage(error, 'Failed to suggest query params'), }); } } From 41eadf1ece8e2a566e08a09396fd005ef9bbfe54 Mon Sep 17 00:00:00 2001 From: zho Date: Mon, 16 Feb 2026 09:13:05 +0800 Subject: [PATCH 03/20] #15-fix for CI Errors --- src/index.ts | 12 ++++++------ src/pinecone-client.test.ts | 4 ++-- src/pinecone-client.ts | 21 ++++++++++++--------- src/server/format-query-result.ts | 14 +++++++------- src/server/tools/generate-urls-tool.ts | 2 +- src/server/url-generation.ts | 12 ++++++------ 6 files changed, 34 insertions(+), 31 deletions(-) diff --git a/src/index.ts b/src/index.ts index fd9a701..0ca2691 100644 --- a/src/index.ts +++ b/src/index.ts @@ -101,17 +101,17 @@ async function main(): Promise { // Set log level (env + logger singleton so tools get correct level) const rawLevel = options.logLevel || - process.env.PINECONE_READ_ONLY_MCP_LOG_LEVEL || - process.env.LOG_LEVEL || + process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || + process.env['LOG_LEVEL'] || 'INFO'; const logLevel = (['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) ? rawLevel : 'INFO') as LogLevel; - process.env.LOG_LEVEL = logLevel; + process.env['LOG_LEVEL'] = logLevel; setLogLevel(logLevel); // Get API key - const apiKey = options.apiKey || process.env.PINECONE_API_KEY; + const apiKey = options.apiKey || process.env['PINECONE_API_KEY']; if (!apiKey) { console.error( 'Error: Pinecone API key is required. Set PINECONE_API_KEY environment variable or use --api-key option.' @@ -120,9 +120,9 @@ async function main(): Promise { } // Get configuration - const indexName = options.indexName || process.env.PINECONE_INDEX_NAME || DEFAULT_INDEX_NAME; + const indexName = options.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; const rerankModel = - options.rerankModel || process.env.PINECONE_RERANK_MODEL || DEFAULT_RERANK_MODEL; + options.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL; // Initialize Pinecone client const client = new PineconeClient({ diff --git a/src/pinecone-client.test.ts b/src/pinecone-client.test.ts index 8b9fa96..4eca78c 100644 --- a/src/pinecone-client.test.ts +++ b/src/pinecone-client.test.ts @@ -32,8 +32,8 @@ describe('PineconeClient', () => { }); it('should use environment variables as fallbacks', () => { - process.env.PINECONE_INDEX_NAME = 'env-index'; - process.env.PINECONE_RERANK_MODEL = 'env-model'; + process.env['PINECONE_INDEX_NAME'] = 'env-index'; + process.env['PINECONE_RERANK_MODEL'] = 'env-model'; const envClient = new PineconeClient({ apiKey: 'test-api-key', diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 5262e2a..836e9e8 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -60,11 +60,13 @@ export class PineconeClient { constructor(config: PineconeClientConfig) { this.apiKey = config.apiKey; - this.indexName = config.indexName || process.env.PINECONE_INDEX_NAME || DEFAULT_INDEX_NAME; + this.indexName = + config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; this.rerankModel = - config.rerankModel || process.env.PINECONE_RERANK_MODEL || DEFAULT_RERANK_MODEL; + config.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL; this.defaultTopK = - config.defaultTopK || parseInt(process.env.PINECONE_TOP_K || String(DEFAULT_TOP_K)); + config.defaultTopK || + parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K)); } /** @@ -214,7 +216,7 @@ export class PineconeClient { // Include filter when explicitly provided (matches Python behavior). if (metadataFilter !== undefined) { - queryPayload.filter = metadataFilter; + queryPayload['filter'] = metadataFilter; if (!options?.fields) { logDebug('Applying metadata filter', metadataFilter); } @@ -243,7 +245,7 @@ export class PineconeClient { }, }; if (metadataFilter !== undefined) { - queryParams.query.filter = metadataFilter; + queryParams.query['filter'] = metadataFilter; } if (options?.fields?.length) { queryParams.fields = options.fields; @@ -270,7 +272,8 @@ export class PineconeClient { const hitId = hit._id || ''; const hitScore = hit._score || 0; - if (hitId in deduped && (deduped[hitId]._score || 0) >= hitScore) { + const existing = deduped[hitId]; + if (existing !== undefined && (existing._score || 0) >= hitScore) { continue; } @@ -456,9 +459,9 @@ export class PineconeClient { const docKeys = new Set(); for (const hit of hits) { const fields = hit.fields || {}; - const docNumber = fields.document_number; - const url = fields.url; - const docId = fields.doc_id; + const docNumber = fields['document_number']; + const url = fields['url']; + const docId = fields['doc_id']; const key = (typeof docNumber === 'string' ? docNumber : undefined) ?? (typeof url === 'string' ? url : undefined) ?? diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts index 19486d1..db7e28e 100644 --- a/src/server/format-query-result.ts +++ b/src/server/format-query-result.ts @@ -36,13 +36,13 @@ export function formatSearchResultAsRow( if (options?.enrichUrls && options?.namespace) { const generated = generateUrlForNamespace(options.namespace, metadata); - if (generated.url && typeof metadata.url !== 'string') { - metadata.url = generated.url; + if (generated.url && typeof metadata['url'] !== 'string') { + metadata['url'] = generated.url; } } - const docNum = metadata.document_number; - const filename = metadata.filename; + const docNum = metadata['document_number']; + const filename = metadata['filename']; const paper_number = (typeof docNum === 'string' ? docNum : null) ?? (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? @@ -50,9 +50,9 @@ export function formatSearchResultAsRow( return { paper_number, - title: String(metadata.title ?? ''), - author: String(metadata.author ?? ''), - url: String(metadata.url ?? ''), + title: String(metadata['title'] ?? ''), + author: String(metadata['author'] ?? ''), + url: String(metadata['url'] ?? ''), content: doc.content.substring(0, contentMaxLength), score: Math.round(doc.score * 10000) / 10000, reranked: doc.reranked, diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts index cb41c75..55130e9 100644 --- a/src/server/tools/generate-urls-tool.ts +++ b/src/server/tools/generate-urls-tool.ts @@ -5,7 +5,7 @@ import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; function extractMetadata(record: Record): Record { - const nested = record.metadata; + const nested = record['metadata']; if (nested && typeof nested === 'object' && !Array.isArray(nested)) { return nested as Record; } diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index c250c05..c79d340 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -12,13 +12,13 @@ export function generateUrlForNamespace( namespace: string, metadata: Record ): UrlGenerationResult { - const existingUrl = asString(metadata.url); + const existingUrl = asString(metadata['url']); if (existingUrl) { return { url: existingUrl, method: 'metadata.url' }; } if (namespace === 'mailing') { - const docIdOrThread = asString(metadata.doc_id) ?? asString(metadata.thread_id); + const docIdOrThread = asString(metadata['doc_id']) ?? asString(metadata['thread_id']); if (!docIdOrThread) { return { url: null, @@ -33,14 +33,14 @@ export function generateUrlForNamespace( } if (namespace === 'slack-Cpplang') { - const source = asString(metadata.source); + const source = asString(metadata['source']); if (source) { return { url: source, method: 'metadata.source' }; } - const teamId = asString(metadata.team_id); - const channelId = asString(metadata.channel_id); - const docId = asString(metadata.doc_id); + const teamId = asString(metadata['team_id']); + const channelId = asString(metadata['channel_id']); + const docId = asString(metadata['doc_id']); if (!teamId || !channelId || !docId) { return { url: null, From 213bb932e3ba9c91d62562c3a6b9eb862f493b61 Mon Sep 17 00:00:00 2001 From: zho Date: Mon, 16 Feb 2026 09:25:47 +0800 Subject: [PATCH 04/20] #15-fix prettier error --- CHANGELOG.md | 4 +- package.json | 1 - src/index.ts | 6 +-- src/logger.ts | 4 +- src/pinecone-client.test.ts | 38 ++++++++++++--- src/pinecone-client.ts | 47 ++++++++++++------- src/server/metadata-filter.ts | 11 ++++- src/server/namespace-router.ts | 2 +- src/server/query-suggestion.ts | 11 ++--- src/server/suggestion-flow.ts | 21 ++++----- src/server/tools/count-tool.ts | 21 +++++++-- src/server/tools/generate-urls-tool.ts | 4 +- src/server/tools/guided-query-tool.ts | 4 +- src/server/tools/namespace-router-tool.ts | 12 ++++- src/server/tools/query-tool.ts | 18 +++---- src/server/tools/suggest-query-params-tool.ts | 2 +- src/server/url-generation.test.ts | 4 +- src/server/url-generation.ts | 7 ++- src/types.ts | 9 +++- tsconfig.json | 16 ++----- 20 files changed, 154 insertions(+), 88 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e87f2aa..369b30f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,8 +20,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - ESLint and Prettier configuration - Complete documentation - - ### Changed - N/A @@ -45,6 +43,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [0.1.1] - 2026-01-27 ### Changed + - Enhanced TypeScript strict mode with additional compiler checks: - Added `noUncheckedIndexedAccess` for safer array/object access - Added `noImplicitOverride` to require explicit override keywords @@ -53,6 +52,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Simplified build script to use standard `tsc` command ### Fixed + - Fixed build script that was suppressing TypeScript compilation errors with `|| exit 0` - Fixed all type safety issues to comply with stricter TypeScript checks diff --git a/package.json b/package.json index 96a3ebe..1b3e8f8 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,6 @@ "node": ">=18.0.0" }, "scripts": { - "clean": "node -e \"const fs=require('fs'); fs.rmSync('dist',{recursive:true,force:true});\"", "build": "npm run clean && node ./node_modules/typescript/bin/tsc", "build:watch": "tsc --watch", diff --git a/src/index.ts b/src/index.ts index 0ca2691..0c9b225 100644 --- a/src/index.ts +++ b/src/index.ts @@ -104,9 +104,9 @@ async function main(): Promise { process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || process.env['LOG_LEVEL'] || 'INFO'; - const logLevel = (['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) - ? rawLevel - : 'INFO') as LogLevel; + const logLevel = ( + ['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) ? rawLevel : 'INFO' + ) as LogLevel; process.env['LOG_LEVEL'] = logLevel; setLogLevel(logLevel); diff --git a/src/logger.ts b/src/logger.ts index d6b01b8..c4de6a6 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -56,6 +56,8 @@ export function warn(msg: string, data?: unknown): void { export function error(msg: string, err?: unknown): void { if (shouldLog('ERROR')) { const detail = err instanceof Error ? err.message : err !== undefined ? String(err) : undefined; - console.error(formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined)); + console.error( + formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined) + ); } } diff --git a/src/pinecone-client.test.ts b/src/pinecone-client.test.ts index 4eca78c..559e43b 100644 --- a/src/pinecone-client.test.ts +++ b/src/pinecone-client.test.ts @@ -66,7 +66,10 @@ describe('PineconeClient', () => { it('should continue hybrid search when one index fails', async () => { const testClient = client as PineconeClientTestDouble; - testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); let searchCall = 0; testClient.searchIndex = async () => { @@ -98,7 +101,10 @@ describe('PineconeClient', () => { it('should throw when both dense and sparse searches fail', async () => { const testClient = client as PineconeClientTestDouble; - testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); testClient.searchIndex = async () => { throw new Error('index failure'); }; @@ -117,15 +123,30 @@ describe('PineconeClient', () => { describe('count', () => { it('should return unique document count using semantic search only with minimal fields', async () => { const testClient = client as PineconeClientTestDouble; - testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); // Two chunks from doc A, one from doc B -> unique count 2 testClient.searchIndex = async (_index, _query, _topK, _ns, _filter, options) => { expect(options?.fields).toEqual(['document_number', 'url', 'doc_id']); return [ - { _id: 'c1', _score: 1, fields: { document_number: 'p1234r0', url: 'https://example.com/1' } }, - { _id: 'c2', _score: 0.9, fields: { document_number: 'p1234r0', url: 'https://example.com/1' } }, - { _id: 'c3', _score: 0.8, fields: { document_number: 'p5678r0', url: 'https://example.com/2' } }, + { + _id: 'c1', + _score: 1, + fields: { document_number: 'p1234r0', url: 'https://example.com/1' }, + }, + { + _id: 'c2', + _score: 0.9, + fields: { document_number: 'p1234r0', url: 'https://example.com/1' }, + }, + { + _id: 'c3', + _score: 0.8, + fields: { document_number: 'p5678r0', url: 'https://example.com/2' }, + }, ]; }; @@ -141,7 +162,10 @@ describe('PineconeClient', () => { it('should set truncated when hit limit is reached', async () => { const testClient = client as PineconeClientTestDouble; - testClient.ensureIndexes = async () => ({ denseIndex: {} as SearchableIndex, sparseIndex: {} as SearchableIndex }); + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); const manyHits: PineconeHit[] = Array.from({ length: 10000 }, (_, i) => ({ _id: `id-${i}`, _score: 1, diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index df2f9c6..98fd9e9 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -60,13 +60,11 @@ export class PineconeClient { constructor(config: PineconeClientConfig) { this.apiKey = config.apiKey; - this.indexName = - config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; + this.indexName = config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; this.rerankModel = config.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL; this.defaultTopK = - config.defaultTopK || - parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K)); + config.defaultTopK || parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K)); } /** @@ -88,7 +86,10 @@ export class PineconeClient { /** * Ensure Pinecone indexes are initialized and return them */ - private async ensureIndexes(): Promise<{ denseIndex: SearchableIndex; sparseIndex: SearchableIndex }> { + private async ensureIndexes(): Promise<{ + denseIndex: SearchableIndex; + sparseIndex: SearchableIndex; + }> { if (this.initialized && this.denseIndex !== null && this.sparseIndex !== null) { return { denseIndex: this.denseIndex, sparseIndex: this.sparseIndex }; } @@ -124,7 +125,9 @@ export class PineconeClient { const { denseIndex } = await this.ensureIndexes(); // Get index stats to find namespaces - const stats = denseIndex.describeIndexStats ? await denseIndex.describeIndexStats() : undefined; + const stats = denseIndex.describeIndexStats + ? await denseIndex.describeIndexStats() + : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; console.error(`Found ${namespaces.length} namespace(s)`); @@ -139,12 +142,18 @@ export class PineconeClient { // Sample a few records to discover metadata fields if (recordCount > 0 && denseIndex.namespace) { try { - const nsObj = denseIndex.namespace(ns) as { query: (opts: { topK: number; vector: number[]; includeMetadata: boolean }) => Promise<{ matches?: Array<{ metadata?: Record }> }> } | null; + const nsObj = denseIndex.namespace(ns) as { + query: (opts: { + topK: number; + vector: number[]; + includeMetadata: boolean; + }) => Promise<{ matches?: Array<{ metadata?: Record }> }>; + } | null; const sampleQuery = nsObj && typeof nsObj.query === 'function' ? await nsObj.query({ topK: 5, - vector: Array((stats?.dimension ?? 1536)).fill(0), + vector: Array(stats?.dimension ?? 1536).fill(0), includeMetadata: true, }) : { matches: undefined }; @@ -225,7 +234,11 @@ export class PineconeClient { try { // Preferred path: Pinecone search API. if (typeof index.search === 'function') { - const searchOpts: { namespace?: string; query: Record; fields?: string[] } = { + const searchOpts: { + namespace?: string; + query: Record; + fields?: string[]; + } = { namespace, query: queryPayload, }; @@ -256,7 +269,9 @@ export class PineconeClient { return result?.result?.hits || []; } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); - throw new Error(`Pinecone search failed for namespace "${namespace ?? 'default'}": ${errorMessage}`); + throw new Error( + `Pinecone search failed for namespace "${namespace ?? 'default'}": ${errorMessage}` + ); } } @@ -319,10 +334,10 @@ export class PineconeClient { query, results as unknown as (string | Record)[], { - topN, - rankFields: ['chunk_text'], - returnDocuments: true, - parameters: { truncate: 'END' }, + topN, + rankFields: ['chunk_text'], + returnDocuments: true, + parameters: { truncate: 'END' }, } ); @@ -385,9 +400,7 @@ export class PineconeClient { // When reranking, Pinecone requires chunk_text in returned fields; add it if user specified fields without it const searchFields = - requestedFields?.length && - useReranking && - !requestedFields.includes('chunk_text') + requestedFields?.length && useReranking && !requestedFields.includes('chunk_text') ? [...requestedFields, 'chunk_text'] : requestedFields; diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index 879a4ea..6bb8c12 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -14,7 +14,16 @@ const metadataFilterValueSchema: z.ZodType = z.lazy(() => ); export const metadataFilterSchema = z.record(z.string(), metadataFilterValueSchema); -const ALLOWED_FILTER_OPERATORS = new Set(['$eq', '$ne', '$gt', '$gte', '$lt', '$lte', '$in', '$nin']); +const ALLOWED_FILTER_OPERATORS = new Set([ + '$eq', + '$ne', + '$gt', + '$gte', + '$lt', + '$lte', + '$in', + '$nin', +]); function isPrimitiveFilterValue(value: unknown): boolean { return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts index bbafe11..4fecf96 100644 --- a/src/server/namespace-router.ts +++ b/src/server/namespace-router.ts @@ -21,7 +21,7 @@ function scoreNamespace( 'wg21-papers': ['wg21', 'paper', 'proposal', 'document_number', 'p0', 'standard'], 'github-compiler': ['github', 'issue', 'pr', 'repository', 'compiler', 'bug'], 'youtube-scripts': ['youtube', 'video', 'script', 'transcript', 'channel'], - 'mailing': ['mailing', 'email', 'thread', 'subject', 'list'], + mailing: ['mailing', 'email', 'thread', 'subject', 'list'], 'slack-cpplang': ['slack', 'chat', 'message', 'channel', 'thread'], 'blog-posts': ['blog', 'post', 'article'], 'cpp-documentation': ['documentation', 'docs', 'reference', 'library'], diff --git a/src/server/query-suggestion.ts b/src/server/query-suggestion.ts index baffbe9..925de66 100644 --- a/src/server/query-suggestion.ts +++ b/src/server/query-suggestion.ts @@ -26,7 +26,8 @@ export function suggestQueryParams( suggested_fields: [], use_count_tool: false, recommended_tool: 'query_fast', - explanation: 'Namespace not found or has no metadata fields. Call list_namespaces first, then pass a valid namespace.', + explanation: + 'Namespace not found or has no metadata fields. Call list_namespaces first, then pass a valid namespace.', namespace_found: false, }; } @@ -48,13 +49,7 @@ export function suggestQueryParams( if ( /\b(content|summarize|summarise|what does|excerpt|text|say|details?|full text|body)\b/.test(q) ) { - const fields = keepOnlyAvailable([ - 'document_number', - 'title', - 'url', - 'author', - 'chunk_text', - ]); + const fields = keepOnlyAvailable(['document_number', 'title', 'url', 'author', 'chunk_text']); return { suggested_fields: fields.length ? fields : available, use_count_tool: false, diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts index d80a19c..3d49b49 100644 --- a/src/server/suggestion-flow.ts +++ b/src/server/suggestion-flow.ts @@ -9,23 +9,22 @@ type FlowState = { const stateByNamespace = new Map(); -export function markSuggested( - namespace: string, - state: Omit -): void { +export function markSuggested(namespace: string, state: Omit): void { stateByNamespace.set(namespace, { ...state, updatedAt: Date.now(), }); } -export function requireSuggested(namespace: string): { - ok: true; - flow: FlowState; -} | { - ok: false; - message: string; -} { +export function requireSuggested(namespace: string): + | { + ok: true; + flow: FlowState; + } + | { + ok: false; + message: string; + } { const state = stateByNamespace.get(namespace); if (!state) { return { diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts index af26624..bc58cde 100644 --- a/src/server/tools/count-tool.ts +++ b/src/server/tools/count-tool.ts @@ -8,7 +8,13 @@ import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; const COUNT_RESPONSE_STATUS = 'success' as const; type CountResponse = - | { status: typeof COUNT_RESPONSE_STATUS; count: number; truncated: boolean; namespace: string; metadata_filter?: Record } + | { + status: typeof COUNT_RESPONSE_STATUS; + count: number; + truncated: boolean; + namespace: string; + metadata_filter?: Record; + } | { status: 'error'; message: string }; export function registerCountTool(server: McpServer): void { @@ -25,10 +31,14 @@ export function registerCountTool(server: McpServer): void { 'For count-by-metadata only, use a broad query_text (e.g. "paper" or "document"). ' + 'Same metadata_filter operators as query: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin.', inputSchema: { - namespace: z.string().describe('Namespace to count in. Use list_namespaces to discover namespaces.'), + namespace: z + .string() + .describe('Namespace to count in. Use list_namespaces to discover namespaces.'), query_text: z .string() - .describe('Search query text. Use a broad term (e.g. "paper", "document") when counting by metadata only.'), + .describe( + 'Search query text. Use a broad term (e.g. "paper", "document") when counting by metadata only.' + ), metadata_filter: metadataFilterSchema .optional() .describe( @@ -40,7 +50,10 @@ export function registerCountTool(server: McpServer): void { try { const { namespace, query_text, metadata_filter } = params; if (!query_text || !query_text.trim()) { - const response: CountResponse = { status: 'error', message: 'query_text cannot be empty' }; + const response: CountResponse = { + status: 'error', + message: 'query_text cannot be empty', + }; return jsonErrorResponse(response); } if (metadata_filter) { diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts index 55130e9..6cd169d 100644 --- a/src/server/tools/generate-urls-tool.ts +++ b/src/server/tools/generate-urls-tool.ts @@ -23,7 +23,9 @@ export function registerGenerateUrlsTool(server: McpServer): void { inputSchema: { namespace: z .string() - .describe('Target namespace. URL generation currently supports mailing and slack-Cpplang.'), + .describe( + 'Target namespace. URL generation currently supports mailing and slack-Cpplang.' + ), records: z .array(z.record(z.string(), z.unknown())) .describe( diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts index 8a9ba37..d027cb5 100644 --- a/src/server/tools/guided-query-tool.ts +++ b/src/server/tools/guided-query-tool.ts @@ -27,7 +27,9 @@ export function registerGuidedQueryTool(server: McpServer): void { namespace: z .string() .optional() - .describe('Optional explicit namespace. If omitted, namespace_router logic will choose one.'), + .describe( + 'Optional explicit namespace. If omitted, namespace_router logic will choose one.' + ), metadata_filter: metadataFilterSchema .optional() .describe('Optional metadata filter to constrain results.'), diff --git a/src/server/tools/namespace-router-tool.ts b/src/server/tools/namespace-router-tool.ts index 65e17e5..95ed513 100644 --- a/src/server/tools/namespace-router-tool.ts +++ b/src/server/tools/namespace-router-tool.ts @@ -13,8 +13,16 @@ export function registerNamespaceRouterTool(server: McpServer): void { 'Suggest likely namespace(s) for a user query using namespace names, metadata fields, and keyword heuristics. ' + 'Use before suggest_query_params when namespace is unclear.', inputSchema: { - user_query: z.string().describe('User question/intent used to infer relevant namespace(s).'), - top_n: z.number().int().min(1).max(5).default(3).describe('Maximum number of suggested namespaces (1-5).'), + user_query: z + .string() + .describe('User question/intent used to infer relevant namespace(s).'), + top_n: z + .number() + .int() + .min(1) + .max(5) + .default(3) + .describe('Maximum number of suggested namespaces (1-5).'), }, }, async (params) => { diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts index c54a4ff..5a728ad 100644 --- a/src/server/tools/query-tool.ts +++ b/src/server/tools/query-tool.ts @@ -23,15 +23,7 @@ type QueryExecParams = { async function executeQuery(params: QueryExecParams) { try { - const { - query_text, - namespace, - top_k, - use_reranking, - metadata_filter, - fields, - mode, - } = params; + const { query_text, namespace, top_k, use_reranking, metadata_filter, fields, mode } = params; if (!query_text || !query_text.trim()) { const response: QueryResponse = { @@ -104,7 +96,9 @@ const baseSchema = { .max(MAX_TOP_K) .default(10) .describe('Number of results to return (1-100). Default: 10'), - metadata_filter: metadataFilterSchema.optional().describe('Optional metadata filter to narrow down search results.'), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to narrow down search results.'), fields: z .array(z.string()) .optional() @@ -125,7 +119,9 @@ export function registerQueryTool(server: McpServer): void { use_reranking: z .boolean() .default(true) - .describe('Whether to use semantic reranking for better relevance. Slower but more accurate.'), + .describe( + 'Whether to use semantic reranking for better relevance. Slower but more accurate.' + ), }, }, async (params) => diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts index f40d8cf..ccb093f 100644 --- a/src/server/tools/suggest-query-params-tool.ts +++ b/src/server/tools/suggest-query-params-tool.ts @@ -11,7 +11,7 @@ export function registerSuggestQueryParamsTool(server: McpServer): void { 'suggest_query_params', { description: - 'Suggest which fields to request and whether to use the count tool, based on the namespace schema (from list_namespaces) and the user\'s natural language query. ' + + "Suggest which fields to request and whether to use the count tool, based on the namespace schema (from list_namespaces) and the user's natural language query. " + 'Call list_namespaces first to get available namespaces and metadata fields. Then call this tool with the target namespace and the user query; ' + 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count/query_fast/query_detailed), and an explanation. ' + 'This step is mandatory before query/count tools; use the returned suggested_fields in query tools to reduce payload and cost.', diff --git a/src/server/url-generation.test.ts b/src/server/url-generation.test.ts index b6a6edf..f717787 100644 --- a/src/server/url-generation.test.ts +++ b/src/server/url-generation.test.ts @@ -25,7 +25,9 @@ describe('generateUrlForNamespace', () => { const r = generateUrlForNamespace('mailing', { thread_id: 'boost@lists.boost.org/thread/ABC123', }); - expect(r.url).toBe('https://lists.boost.org/archives/list/boost@lists.boost.org/thread/ABC123/'); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost@lists.boost.org/thread/ABC123/' + ); expect(r.method).toBe('generated.mailing'); }); diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index c79d340..7381494 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -1,6 +1,11 @@ export type UrlGenerationResult = { url: string | null; - method: 'metadata.url' | 'metadata.source' | 'generated.mailing' | 'generated.slack' | 'unavailable'; + method: + | 'metadata.url' + | 'metadata.source' + | 'generated.mailing' + | 'generated.slack' + | 'unavailable'; reason?: string; }; diff --git a/src/types.ts b/src/types.ts index 007ef8e..9040b8b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -98,7 +98,10 @@ export interface MergedHit { /** Minimal index interface used for hybrid search (dense/sparse) and namespace discovery. */ export interface SearchableIndex { - describeIndexStats?(): Promise<{ dimension?: number; namespaces?: Record }>; + describeIndexStats?(): Promise<{ + dimension?: number; + namespaces?: Record; + }>; search?(opts: { namespace?: string; query: Record; @@ -109,5 +112,7 @@ export interface SearchableIndex { matches?: Array<{ metadata?: Record }>; }>; }; - searchRecords?(params: { query: Record }): Promise<{ result?: { hits?: PineconeHit[] } }>; + searchRecords?(params: { + query: Record; + }): Promise<{ result?: { hits?: PineconeHit[] } }>; } diff --git a/tsconfig.json b/tsconfig.json index 9b41d5e..b63c222 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -16,16 +16,8 @@ "declaration": true, "declarationMap": true, "sourceMap": true, - "typeRoots": [ - "./node_modules/@types" - ] + "typeRoots": ["./node_modules/@types"] }, - "include": [ - "src/**/*" - ], - "exclude": [ - "node_modules", - "dist", - "**/*.test.ts" - ] -} \ No newline at end of file + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts"] +} From 10a521fe3ac544831f88de8e09c9fee4e3d0fe27 Mon Sep 17 00:00:00 2001 From: zho Date: Fri, 20 Feb 2026 23:48:50 +0800 Subject: [PATCH 05/20] #15-fix errors for first coderabbit review --- .github/workflows/ci.yml | 4 +- src/constants.ts | 13 +++- src/logger.ts | 11 ++- src/pinecone-client.ts | 56 +++++++------- src/server.ts | 2 + src/server/format-query-result.ts | 9 ++- src/server/metadata-filter.ts | 7 ++ src/server/namespace-router.ts | 35 ++++----- src/server/suggestion-flow.ts | 14 ++++ src/server/tool-response.ts | 2 +- src/server/tools/count-tool.ts | 2 +- src/server/tools/generate-urls-tool.ts | 5 +- src/server/tools/guided-query-tool.ts | 8 +- src/server/tools/namespace-router-tool.ts | 2 +- src/server/tools/query-tool.ts | 5 +- src/server/url-generation.test.ts | 16 +++- src/server/url-generation.ts | 94 ++++++++++++++--------- src/types.ts | 28 +++++-- 18 files changed, 202 insertions(+), 111 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a560ab..0d97076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: branches: [main] push: branches: [main] - workflow_call: # Allow this workflow to be called by other workflows + workflow_call: # Allow this workflow to be called by other workflows jobs: build-and-test: @@ -39,7 +39,7 @@ jobs: run: npm run build - name: Smoke test CLI - run: npm run smoke + run: node dist/index.js --help - name: Run tests run: npm test diff --git a/src/constants.ts b/src/constants.ts index 57227b9..397cf41 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -15,7 +15,11 @@ export const COUNT_TOP_K = 10_000; export const COUNT_FIELDS = ['document_number', 'url', 'doc_id'] as const; /** Default lightweight field set for fast queries. */ export const FAST_QUERY_FIELDS = ['document_number', 'title', 'url', 'author', 'doc_id'] as const; -export const DEFAULT_NAMESPACE = 'mailing'; +/** query_documents: default and max number of documents to return (reassembled from chunks). */ +export const DEFAULT_QUERY_DOCUMENTS_TOP_K = 5; +export const MAX_QUERY_DOCUMENTS_TOP_K = 20; +/** Max chunk hits to fetch when reassembling documents (then group by document). */ +export const QUERY_DOCUMENTS_MAX_CHUNKS = 500; export const SERVER_NAME = 'Pinecone Read-Only MCP'; export const SERVER_VERSION = '0.1.0'; @@ -29,10 +33,11 @@ Features: - Metadata Filtering: Supports optional metadata filters for refined searches - Namespace Router: Suggests likely namespace(s) from natural-language intent - Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count. -- URL Generation: Use generate_urls to synthesize URLs for mailing/slack records when metadata lacks url. +- URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url. +- Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization. Usage: 1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index 2. Optionally use namespace_router to choose candidate namespace(s) from user intent -3. Call suggest_query_params before query/count tools (mandatory flow gate) to get suggested_fields and recommended tool -4. Use count for count questions, query_fast for lightweight retrieval, or query_detailed/query for content-heavy retrieval`; +3. Call suggest_query_params before query/count/query_documents tools (mandatory flow gate) to get suggested_fields and recommended tool +4. Use count for count questions, query_fast/query_detailed for chunk-level retrieval, or query_documents for full-document content`; diff --git a/src/logger.ts b/src/logger.ts index c4de6a6..0e940c0 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -55,7 +55,16 @@ export function warn(msg: string, data?: unknown): void { export function error(msg: string, err?: unknown): void { if (shouldLog('ERROR')) { - const detail = err instanceof Error ? err.message : err !== undefined ? String(err) : undefined; + // const detail = err instanceof Error ? err.message : err !== undefined ? String(err) : undefined; + // console.error( + // formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined) + // ); + const detail = + err instanceof Error + ? { message: err.message, stack: err.stack } + : err !== undefined + ? String(err) + : undefined; console.error( formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined) ); diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 98fd9e9..385c72e 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -7,7 +7,7 @@ */ import { Pinecone } from '@pinecone-database/pinecone'; -import { debug as logDebug, error as logError, info as logInfo } from './logger.js'; +import { debug as logDebug, error as logError, info as logInfo, warn as logWarn } from './logger.js'; import type { PineconeClientConfig, SearchResult, @@ -16,6 +16,7 @@ import type { CountParams, CountResult, MergedHit, + NamespaceHandle, SearchableIndex, PineconeMetadataValue, } from './types.js'; @@ -130,7 +131,7 @@ export class PineconeClient { : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; - console.error(`Found ${namespaces.length} namespace(s)`); + logInfo(`Found ${namespaces.length} namespace(s)`); // Get metadata info for each namespace by sampling records const namespacesInfo = await Promise.all( @@ -142,15 +143,9 @@ export class PineconeClient { // Sample a few records to discover metadata fields if (recordCount > 0 && denseIndex.namespace) { try { - const nsObj = denseIndex.namespace(ns) as { - query: (opts: { - topK: number; - vector: number[]; - includeMetadata: boolean; - }) => Promise<{ matches?: Array<{ metadata?: Record }> }>; - } | null; + const nsObj: NamespaceHandle = denseIndex.namespace(ns); const sampleQuery = - nsObj && typeof nsObj.query === 'function' + typeof nsObj.query === 'function' ? await nsObj.query({ topK: 5, vector: Array(stats?.dimension ?? 1536).fill(0), @@ -178,7 +173,7 @@ export class PineconeClient { }); } } catch (queryError) { - console.error(`Error sampling records for namespace ${ns}:`, queryError); + logError(`Error sampling records for namespace ${ns}`, queryError); } } @@ -188,7 +183,7 @@ export class PineconeClient { metadata: metadataFields, }; } catch (error) { - console.error(`Error processing namespace ${ns}:`, error); + logError(`Error processing namespace ${ns}`, error); return { namespace: ns, recordCount: 0, @@ -200,7 +195,7 @@ export class PineconeClient { return namespacesInfo; } catch (error) { - console.error('Error listing namespaces:', error); + logError('Error listing namespaces', error); return []; } } @@ -391,11 +386,9 @@ export class PineconeClient { if (topK < 1) { throw new Error('topK must be at least 1'); } - // Allow up to COUNT_TOP_K when explicitly requested (e.g. for count tool); otherwise cap at MAX_TOP_K - const maxAllowed = - requestedTopK !== undefined && requestedTopK > MAX_TOP_K ? COUNT_TOP_K : MAX_TOP_K; - if (topK > maxAllowed) { - topK = maxAllowed; + + if (topK > MAX_TOP_K) { + topK = MAX_TOP_K; } // When reranking, Pinecone requires chunk_text in returned fields; add it if user specified fields without it @@ -419,10 +412,10 @@ export class PineconeClient { const sparseHits = sparseResult.status === 'fulfilled' ? sparseResult.value : []; if (denseResult.status === 'rejected') { - console.error('Dense index search failed:', denseResult.reason); + logError('Dense index search failed', denseResult.reason); } if (sparseResult.status === 'rejected') { - console.error('Sparse index search failed:', sparseResult.reason); + logError('Sparse index search failed', sparseResult.reason); } if (denseResult.status === 'rejected' && sparseResult.status === 'rejected') { throw new Error('Hybrid search failed: both dense and sparse index searches failed.'); @@ -445,7 +438,7 @@ export class PineconeClient { })); } - console.error( + logInfo( `Retrieved ${documents.length} documents from hybrid search (dense: ${denseHits.length}, sparse: ${sparseHits.length})` ); @@ -470,18 +463,29 @@ export class PineconeClient { ); const docKeys = new Set(); + let idFallbackCount = 0; for (const hit of hits) { const fields = hit.fields || {}; const docNumber = fields['document_number']; const url = fields['url']; const docId = fields['doc_id']; - const key = + const docKey = (typeof docNumber === 'string' ? docNumber : undefined) ?? (typeof url === 'string' ? url : undefined) ?? - (typeof docId === 'string' ? docId : undefined) ?? - hit._id ?? - ''; - docKeys.add(key); + (typeof docId === 'string' ? docId : undefined); + if (docKey !== undefined) { + docKeys.add(docKey); + } else { + // Fall back to chunk ID — this yields a chunk count, not a document count + idFallbackCount++; + docKeys.add(hit._id ?? ''); + } + } + if (idFallbackCount > 0) { + logWarn( + `count(): ${idFallbackCount} hit(s) in namespace "${params.namespace}" had none of the ` + + `identifier fields (${COUNT_FIELDS.join(', ')}); fell back to chunk ID — result may overcount documents` + ); } const count = docKeys.size; diff --git a/src/server.ts b/src/server.ts index e44f0eb..61e0d7f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -12,6 +12,7 @@ import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js'; import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js'; import { registerListNamespacesTool } from './server/tools/list-namespaces-tool.js'; import { registerNamespaceRouterTool } from './server/tools/namespace-router-tool.js'; +import { registerQueryDocumentsTool } from './server/tools/query-documents-tool.js'; import { registerQueryTool } from './server/tools/query-tool.js'; import { registerSuggestQueryParamsTool } from './server/tools/suggest-query-params-tool.js'; @@ -36,6 +37,7 @@ export async function setupServer(): Promise { registerSuggestQueryParamsTool(server); registerCountTool(server); registerQueryTool(server); + registerQueryDocumentsTool(server); registerGuidedQueryTool(server); registerGenerateUrlsTool(server); diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts index db7e28e..0c6f72c 100644 --- a/src/server/format-query-result.ts +++ b/src/server/format-query-result.ts @@ -21,7 +21,7 @@ export interface QueryResultRow { /** * Format a single search result into a QueryResponse result row. - * Optionally enrich url from namespace (mailing/slack-Cpplang) when metadata.url is missing. + * Optionally enrich url using the namespace URL generator when metadata.url is missing (if supported). */ export function formatSearchResultAsRow( doc: SearchResult, @@ -36,7 +36,10 @@ export function formatSearchResultAsRow( if (options?.enrichUrls && options?.namespace) { const generated = generateUrlForNamespace(options.namespace, metadata); - if (generated.url && typeof metadata['url'] !== 'string') { + const existingUrl = metadata['url']; + const urlIsBlank = + typeof existingUrl !== 'string' || existingUrl.trim() === ''; + if (generated.url && urlIsBlank) { metadata['url'] = generated.url; } } @@ -45,7 +48,7 @@ export function formatSearchResultAsRow( const filename = metadata['filename']; const paper_number = (typeof docNum === 'string' ? docNum : null) ?? - (typeof filename === 'string' ? filename.replace('.md', '').toUpperCase() : null) ?? + (typeof filename === 'string' ? filename.replace(/\.md$/i, '').toUpperCase() : null) ?? null; return { diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index 6bb8c12..a02c66b 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -54,6 +54,13 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n if ((key === '$in' || key === '$nin') && !isPrimitiveArray(nestedValue)) { return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; } + if ( + (key === '$eq' || key === '$ne' || key === '$gt' || key === '$gte' || + key === '$lt' || key === '$lte') && + !isPrimitiveFilterValue(nestedValue) + ) { + return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; + } } const nestedError = validateMetadataFilterValue(nestedValue, [...path, key]); diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts index 4fecf96..72baf35 100644 --- a/src/server/namespace-router.ts +++ b/src/server/namespace-router.ts @@ -7,6 +7,12 @@ type RankedNamespace = { reasons: string[]; }; +/** + * Score a namespace for relevance to the query using only: + * - Query containing the namespace name (normalized) + * - Query containing any of the namespace's metadata field names + * No hardcoded namespace names or keyword lists; works for any index/namespace. + */ function scoreNamespace( query: string, namespace: string, @@ -17,26 +23,17 @@ function scoreNamespace( const reasons: string[] = []; let score = 0; - const keywordHints: Record = { - 'wg21-papers': ['wg21', 'paper', 'proposal', 'document_number', 'p0', 'standard'], - 'github-compiler': ['github', 'issue', 'pr', 'repository', 'compiler', 'bug'], - 'youtube-scripts': ['youtube', 'video', 'script', 'transcript', 'channel'], - mailing: ['mailing', 'email', 'thread', 'subject', 'list'], - 'slack-cpplang': ['slack', 'chat', 'message', 'channel', 'thread'], - 'blog-posts': ['blog', 'post', 'article'], - 'cpp-documentation': ['documentation', 'docs', 'reference', 'library'], - }; - - if (q.includes(name.replace(/[^a-z0-9]/g, ' '))) { + const normalizedName = name.replace(/[^a-z0-9]/g, ' ').trim(); + if (normalizedName && q.includes(normalizedName)) { score += 3; reasons.push('query mentions namespace name'); - } - - const hints = keywordHints[name] ?? []; - for (const hint of hints) { - if (q.includes(hint)) { - score += 2; - reasons.push(`keyword match: ${hint}`); + } else { + const nameTokens = normalizedName.split(/\s+/).filter(Boolean); + for (const token of nameTokens) { + if (token.length >= 2 && q.includes(token)) { + score += 2; + reasons.push(`query matches namespace token: ${token}`); + } } } @@ -68,6 +65,8 @@ export function rankNamespacesByQuery( }) .sort((a, b) => { if (b.score !== a.score) return b.score - a.score; + // On equal score, prefer the smaller (more-specific) namespace so that + // targeted namespaces are chosen over large catch-all ones. return a.record_count - b.record_count; }) .slice(0, topN); diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts index 3d49b49..ecf12f5 100644 --- a/src/server/suggestion-flow.ts +++ b/src/server/suggestion-flow.ts @@ -9,7 +9,21 @@ type FlowState = { const stateByNamespace = new Map(); +/** + * Evict all entries older than FLOW_CACHE_TTL_MS. + * Called on every write so the map stays bounded without a background timer. + */ +function sweepExpired(): void { + const now = Date.now(); + for (const [ns, state] of stateByNamespace) { + if (now - state.updatedAt > FLOW_CACHE_TTL_MS) { + stateByNamespace.delete(ns); + } + } +} + export function markSuggested(namespace: string, state: Omit): void { + sweepExpired(); stateByNamespace.set(namespace, { ...state, updatedAt: Date.now(), diff --git a/src/server/tool-response.ts b/src/server/tool-response.ts index 1ecfec7..c127bff 100644 --- a/src/server/tool-response.ts +++ b/src/server/tool-response.ts @@ -1,4 +1,4 @@ -type TextPayload = { +export type TextPayload = { content: Array<{ type: 'text'; text: string }>; isError?: boolean; }; diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts index bc58cde..6a798ee 100644 --- a/src/server/tools/count-tool.ts +++ b/src/server/tools/count-tool.ts @@ -49,7 +49,7 @@ export function registerCountTool(server: McpServer): void { async (params) => { try { const { namespace, query_text, metadata_filter } = params; - if (!query_text || !query_text.trim()) { + if (!query_text.trim()) { const response: CountResponse = { status: 'error', message: 'query_text cannot be empty', diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts index 6cd169d..b39b697 100644 --- a/src/server/tools/generate-urls-tool.ts +++ b/src/server/tools/generate-urls-tool.ts @@ -18,13 +18,12 @@ export function registerGenerateUrlsTool(server: McpServer): void { { description: 'Generate URLs for retrieved results when metadata does not include url and URL is needed. ' + - 'Supported namespaces: mailing and slack-Cpplang. For mailing, URL is generated from doc_id or thread_id. ' + - 'For slack-Cpplang, source is preferred when present; otherwise URL is generated from team_id, channel_id, and doc_id.', + 'Uses the URL generator registered for the given namespace (if any); returns unavailable for namespaces without a generator.', inputSchema: { namespace: z .string() .describe( - 'Target namespace. URL generation currently supports mailing and slack-Cpplang.' + 'Target namespace. URL generation is supported only for namespaces that have a registered generator (call list_namespaces to discover namespaces).' ), records: z .array(z.record(z.string(), z.unknown())) diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts index d027cb5..4a17b9d 100644 --- a/src/server/tools/guided-query-tool.ts +++ b/src/server/tools/guided-query-tool.ts @@ -48,7 +48,7 @@ export function registerGuidedQueryTool(server: McpServer): void { .boolean() .default(true) .describe( - 'If true, enrich result URLs for mailing/slack-Cpplang when metadata.url is missing.' + 'If true, enrich result URLs using the namespace URL generator when metadata.url is missing (if supported for that namespace).' ), }, }, @@ -58,9 +58,9 @@ export function registerGuidedQueryTool(server: McpServer): void { user_query, namespace: inputNamespace, metadata_filter, - top_k = 10, - preferred_tool = 'auto', - enrich_urls = true, + top_k, + preferred_tool, + enrich_urls, } = params; if (!user_query?.trim()) { diff --git a/src/server/tools/namespace-router-tool.ts b/src/server/tools/namespace-router-tool.ts index 95ed513..25e48cc 100644 --- a/src/server/tools/namespace-router-tool.ts +++ b/src/server/tools/namespace-router-tool.ts @@ -27,7 +27,7 @@ export function registerNamespaceRouterTool(server: McpServer): void { }, async (params) => { try { - const { user_query, top_n = 3 } = params; + const { user_query, top_n } = params; if (!user_query?.trim()) { return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); } diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts index 5a728ad..81ffcd5 100644 --- a/src/server/tools/query-tool.ts +++ b/src/server/tools/query-tool.ts @@ -22,9 +22,8 @@ type QueryExecParams = { }; async function executeQuery(params: QueryExecParams) { + const { query_text, namespace, top_k, use_reranking, metadata_filter, fields, mode } = params; try { - const { query_text, namespace, top_k, use_reranking, metadata_filter, fields, mode } = params; - if (!query_text || !query_text.trim()) { const response: QueryResponse = { status: 'error', @@ -73,7 +72,7 @@ async function executeQuery(params: QueryExecParams) { }; return jsonResponse(response); } catch (error) { - logToolError('query', error); + logToolError(mode, error); const response: QueryResponse = { status: 'error', message: getToolErrorMessage(error, 'An error occurred while processing your query'), diff --git a/src/server/url-generation.test.ts b/src/server/url-generation.test.ts index f717787..ffe9a03 100644 --- a/src/server/url-generation.test.ts +++ b/src/server/url-generation.test.ts @@ -48,7 +48,7 @@ describe('generateUrlForNamespace', () => { channel_id: 'C123456', doc_id: '1234567.890', }); - expect(r.url).toBe('https://app.slack.com/T123456789/C123456/p1234567890'); + expect(r.url).toBe('https://app.slack.com/client/T123456789/C123456/p1234567890'); expect(r.method).toBe('generated.slack'); }); @@ -57,4 +57,18 @@ describe('generateUrlForNamespace', () => { expect(r.url).toBeNull(); expect(r.method).toBe('unavailable'); }); + it('returns unavailable for mailing when no doc_id or thread_id', () => { + const r = generateUrlForNamespace('mailing', { author: 'someone' }); + expect(r.url).toBeNull(); + expect(r.method).toBe('unavailable'); + }); + + it('returns unavailable for slack-Cpplang when required fields are missing', () => { + const r = generateUrlForNamespace('slack-Cpplang', { + team_id: 'T123', + // channel_id missing, doc_id missing, no source + }); + expect(r.url).toBeNull(); + expect(r.method).toBe('unavailable'); + }); }); diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index 7381494..2650952 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -9,10 +9,59 @@ export type UrlGenerationResult = { reason?: string; }; +type UrlGenerator = (metadata: Record) => UrlGenerationResult; + +/** Registry of namespace -> URL generator. Built-ins are registered below; more can be added at runtime. */ +const urlGenerators = new Map(); + function asString(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; } +function generatorMailing(metadata: Record): UrlGenerationResult { + const docIdOrThread = asString(metadata['doc_id']) ?? asString(metadata['thread_id']); + if (!docIdOrThread) { + return { + url: null, + method: 'unavailable', + reason: 'mailing requires doc_id or thread_id to generate URL', + }; + } + return { + url: `https://lists.boost.org/archives/list/${docIdOrThread}/`, + method: 'generated.mailing', + }; +} + +function generatorSlackCpplang(metadata: Record): UrlGenerationResult { + const source = asString(metadata['source']); + if (source) { + return { url: source, method: 'metadata.source' }; + } + const teamId = asString(metadata['team_id']); + const channelId = asString(metadata['channel_id']); + const docId = asString(metadata['doc_id']); + if (!teamId || !channelId || !docId) { + return { + url: null, + method: 'unavailable', + reason: 'slack-Cpplang requires team_id, channel_id, and doc_id (or source)', + }; + } + const messageId = docId.replace(/\./g, ''); + return { + url: `https://app.slack.com/client/${teamId}/${channelId}/p${messageId}`, + method: 'generated.slack', + }; +} + +urlGenerators.set('mailing', generatorMailing); +urlGenerators.set('slack-Cpplang', generatorSlackCpplang); + +/** + * Generate a URL for a record in the given namespace when metadata.url is missing. + * Uses the registry of URL generators; returns unavailable for namespaces without a generator. + */ export function generateUrlForNamespace( namespace: string, metadata: Record @@ -22,43 +71,9 @@ export function generateUrlForNamespace( return { url: existingUrl, method: 'metadata.url' }; } - if (namespace === 'mailing') { - const docIdOrThread = asString(metadata['doc_id']) ?? asString(metadata['thread_id']); - if (!docIdOrThread) { - return { - url: null, - method: 'unavailable', - reason: 'mailing requires doc_id or thread_id to generate URL', - }; - } - return { - url: `https://lists.boost.org/archives/list/${docIdOrThread}/`, - method: 'generated.mailing', - }; - } - - if (namespace === 'slack-Cpplang') { - const source = asString(metadata['source']); - if (source) { - return { url: source, method: 'metadata.source' }; - } - - const teamId = asString(metadata['team_id']); - const channelId = asString(metadata['channel_id']); - const docId = asString(metadata['doc_id']); - if (!teamId || !channelId || !docId) { - return { - url: null, - method: 'unavailable', - reason: 'slack-Cpplang requires team_id, channel_id, and doc_id (or source)', - }; - } - - const messageId = docId.replace(/\./g, ''); - return { - url: `https://app.slack.com/${teamId}/${channelId}/p${messageId}`, - method: 'generated.slack', - }; + const generator = urlGenerators.get(namespace); + if (generator) { + return generator(metadata); } return { @@ -67,3 +82,8 @@ export function generateUrlForNamespace( reason: `URL generation is not supported for namespace "${namespace}"`, }; } + +/** Register a URL generator for a namespace (e.g. for custom indexes). */ +export function registerUrlGenerator(namespace: string, fn: UrlGenerator): void { + urlGenerators.set(namespace, fn); +} diff --git a/src/types.ts b/src/types.ts index 9040b8b..485e8a1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -96,7 +96,25 @@ export interface MergedHit { metadata: Record; } -/** Minimal index interface used for hybrid search (dense/sparse) and namespace discovery. */ +/** + * Handle for a specific namespace, returned by SearchableIndex.namespace(). + * Carries the legacy query() path (vector-based metadata sampling) and the + * backward-compatible searchRecords() fallback. + */ +export interface NamespaceHandle { + query?(opts: { topK: number; vector: number[]; includeMetadata: boolean }): Promise<{ + matches?: Array<{ metadata?: Record }>; + }>; + searchRecords?(params: { + query: Record; + }): Promise<{ result?: { hits?: PineconeHit[] } }>; +} + +/** + * Minimal top-level index interface for hybrid search (dense/sparse) and namespace discovery. + * Methods are optional because the object is obtained via an `as unknown as` cast from the + * Pinecone SDK, whose concrete shape can vary across SDK versions. + */ export interface SearchableIndex { describeIndexStats?(): Promise<{ dimension?: number; @@ -107,11 +125,9 @@ export interface SearchableIndex { query: Record; fields?: string[]; }): Promise<{ result?: { hits?: PineconeHit[] } }>; - namespace?(name: string): SearchableIndex & { - query?(opts: { topK: number; vector: number[]; includeMetadata: boolean }): Promise<{ - matches?: Array<{ metadata?: Record }>; - }>; - }; + /** Return a namespace-scoped handle for metadata sampling or legacy record queries. */ + namespace?(name: string): NamespaceHandle; + /** Backward-compatible fallback when the SDK exposes searchRecords on the top-level index. */ searchRecords?(params: { query: Record; }): Promise<{ result?: { hits?: PineconeHit[] } }>; From 2d17b9279aa4f6df6ed0754a16691da1778e5e18 Mon Sep 17 00:00:00 2001 From: zho Date: Fri, 20 Feb 2026 23:53:14 +0800 Subject: [PATCH 06/20] #15-add a new tool that re-assembled from the chunk for detail content analysis --- src/server/reassemble-documents.test.ts | 88 ++++++++++++++++ src/server/reassemble-documents.ts | 104 ++++++++++++++++++ src/server/tools/query-documents-tool.ts | 129 +++++++++++++++++++++++ 3 files changed, 321 insertions(+) create mode 100644 src/server/reassemble-documents.test.ts create mode 100644 src/server/reassemble-documents.ts create mode 100644 src/server/tools/query-documents-tool.ts diff --git a/src/server/reassemble-documents.test.ts b/src/server/reassemble-documents.test.ts new file mode 100644 index 0000000..33448eb --- /dev/null +++ b/src/server/reassemble-documents.test.ts @@ -0,0 +1,88 @@ +import { describe, it, expect } from 'vitest'; +import { reassembleByDocument } from './reassemble-documents.js'; +import type { SearchResult } from '../types.js'; + +describe('reassembleByDocument', () => { + it('groups chunks by document_number', () => { + const results: SearchResult[] = [ + { + id: 'c1', + content: 'First chunk.', + score: 0.9, + metadata: { document_number: 'P1234', chunk_index: 0 }, + reranked: false, + }, + { + id: 'c2', + content: 'Second chunk.', + score: 0.8, + metadata: { document_number: 'P1234', chunk_index: 1 }, + reranked: false, + }, + { + id: 'c3', + content: 'Other doc.', + score: 0.7, + metadata: { document_number: 'P5678' }, + reranked: false, + }, + ]; + const docs = reassembleByDocument(results); + expect(docs).toHaveLength(2); + const p1234 = docs.find((d) => d.document_id === 'P1234'); + const p5678 = docs.find((d) => d.document_id === 'P5678'); + expect(p1234?.merged_content).toBe('First chunk.\n\nSecond chunk.'); + expect(p1234?.chunk_count).toBe(2); + expect(p5678?.merged_content).toBe('Other doc.'); + expect(p5678?.chunk_count).toBe(1); + }); + + it('sorts chunks by chunk_index when present', () => { + const results: SearchResult[] = [ + { + id: 'b', + content: 'Second', + score: 0.5, + metadata: { document_number: 'D1', chunk_index: 1 }, + reranked: false, + }, + { + id: 'a', + content: 'First', + score: 0.9, + metadata: { document_number: 'D1', chunk_index: 0 }, + reranked: false, + }, + ]; + const docs = reassembleByDocument(results); + expect(docs[0].merged_content).toBe('First\n\nSecond'); + }); + + it('uses doc_id when document_number is missing', () => { + const results: SearchResult[] = [ + { + id: 'x', + content: 'Content', + score: 0.8, + metadata: { doc_id: 'my-doc-1' }, + reranked: false, + }, + ]; + const docs = reassembleByDocument(results); + expect(docs[0].document_id).toBe('my-doc-1'); + }); + + it('respects maxChunksPerDocument', () => { + const results: SearchResult[] = Array.from({ length: 10 }, (_, i) => ({ + id: `c${i}`, + content: `Chunk ${i}`, + score: 0.9 - i * 0.01, + metadata: { document_number: 'P1', chunk_index: i }, + reranked: false, + })); + const docs = reassembleByDocument(results, { maxChunksPerDocument: 3 }); + expect(docs).toHaveLength(1); + expect(docs[0].chunk_count).toBe(3); + expect(docs[0].merged_content).toBe('Chunk 0\n\nChunk 1\n\nChunk 2'); + }); +}); diff --git a/src/server/reassemble-documents.ts b/src/server/reassemble-documents.ts new file mode 100644 index 0000000..f11522a --- /dev/null +++ b/src/server/reassemble-documents.ts @@ -0,0 +1,104 @@ +/** + * Reassemble chunk-level search results into document-level results. + * Groups by document identity (document_number / doc_id / url) and merges chunk content + * for content analysis (summarization, full-document Q&A, etc.). + */ + +import type { PineconeMetadataValue, SearchResult } from '../types.js'; + +/** Default metadata keys tried for chunk ordering (RecursiveCharacterTextSplitter often adds these). */ +const CHUNK_ORDER_KEYS = ['chunk_index', 'chunk_index_0', 'index', 'loc'] as const; + +function getDocumentKey(hit: SearchResult): string { + const m = hit.metadata || {}; + const docNumber = m['document_number']; + const url = m['url']; + const docId = m['doc_id']; + return ( + (typeof docNumber === 'string' ? docNumber : undefined) ?? + (typeof url === 'string' ? url : undefined) ?? + (typeof docId === 'string' ? docId : undefined) ?? + hit.id ?? + '' + ); +} + +function getChunkOrder(metadata: Record): number { + for (const key of CHUNK_ORDER_KEYS) { + const v = metadata[key]; + if (typeof v === 'number' && Number.isFinite(v)) return v; + if (typeof v === 'string' && /^\d+$/.test(v)) return parseInt(v, 10); + } + return -1; +} + +export interface ReassembledDocument { + /** Document identity (document_number, doc_id, or url). */ + document_id: string; + /** Merged content from all chunks (ordered by chunk_index when available). */ + merged_content: string; + /** Metadata from the first chunk (title, author, url, etc.). */ + metadata: Record; + /** Number of chunks merged. */ + chunk_count: number; + /** Best score among chunks (for ranking documents). */ + best_score: number; +} + +/** + * Group search results by document and merge chunk content. + * Chunks are ordered by metadata chunk_index (or similar) when present; otherwise retrieval order. + */ +export function reassembleByDocument( + results: SearchResult[], + options?: { + /** Max chunks to merge per document (avoids huge payloads). Default 200. */ + maxChunksPerDocument?: number; + /** Separator between chunk contents. Default double newline. */ + contentSeparator?: string; + } +): ReassembledDocument[] { + const maxChunks = options?.maxChunksPerDocument ?? 200; + const separator = options?.contentSeparator ?? '\n\n'; + + const byDoc = new Map(); + + for (const hit of results) { + const key = getDocumentKey(hit); + if (!key) continue; + let list = byDoc.get(key); + if (!list) { + list = []; + byDoc.set(key, list); + } + list.push(hit); + } + + const out: ReassembledDocument[] = []; + + for (const [docId, chunks] of byDoc) { + const sorted = [...chunks].sort((a, b) => { + const orderA = getChunkOrder(a.metadata ?? {}); + const orderB = getChunkOrder(b.metadata ?? {}); + if (orderA >= 0 && orderB >= 0) return orderA - orderB; + if (orderA >= 0) return -1; + if (orderB >= 0) return 1; + return 0; + }); + + const toMerge = sorted.slice(0, maxChunks); + const merged_content = toMerge.map((c) => c.content.trim()).filter(Boolean).join(separator); + const first = toMerge[0]; + const best_score = Math.max(...toMerge.map((c) => c.score)); + + out.push({ + document_id: docId, + merged_content, + metadata: (first?.metadata ?? {}) as Record, + chunk_count: toMerge.length, + best_score: Math.round(best_score * 10000) / 10000, + }); + } + + return out; +} diff --git a/src/server/tools/query-documents-tool.ts b/src/server/tools/query-documents-tool.ts new file mode 100644 index 0000000..896e256 --- /dev/null +++ b/src/server/tools/query-documents-tool.ts @@ -0,0 +1,129 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { + DEFAULT_QUERY_DOCUMENTS_TOP_K, + MAX_QUERY_DOCUMENTS_TOP_K, + QUERY_DOCUMENTS_MAX_CHUNKS, +} from '../../constants.js'; +import { getPineconeClient } from '../client-context.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { reassembleByDocument } from '../reassemble-documents.js'; +import { requireSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** Approximate chunks to fetch per requested document (to get enough for reassembly). */ +const CHUNKS_PER_DOCUMENT = 50; + +export function registerQueryDocumentsTool(server: McpServer): void { + server.registerTool( + 'query_documents', + { + description: + 'Run a semantic query and return whole documents (reassembled from chunks). ' + + 'Use for content analysis, summarization, or when you need full-document context. ' + + 'Chunks are grouped by document_number/doc_id/url, ordered by chunk_index when present (e.g. from RecursiveCharacterTextSplitter), and merged into one content per document. ' + + 'Mandatory flow: call suggest_query_params first. Use list_namespaces to discover namespaces.', + inputSchema: { + query_text: z.string().describe('Search query text. Be specific for better results.'), + namespace: z + .string() + .describe( + 'Namespace to search. Use list_namespaces/namespace_router first, then suggest_query_params.' + ), + top_k: z + .number() + .int() + .min(1) + .max(MAX_QUERY_DOCUMENTS_TOP_K) + .default(DEFAULT_QUERY_DOCUMENTS_TOP_K) + .describe( + `Number of documents to return (1-${MAX_QUERY_DOCUMENTS_TOP_K}). Each document is reassembled from its chunks. Default: ${DEFAULT_QUERY_DOCUMENTS_TOP_K}.` + ), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to narrow search.'), + max_chunks_per_document: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe( + 'Max chunks to merge per document (default 200). Lower for shorter merged_content.' + ), + }, + }, + async (params) => { + try { + const { + query_text, + namespace, + top_k = DEFAULT_QUERY_DOCUMENTS_TOP_K, + metadata_filter, + max_chunks_per_document, + } = params; + + if (!query_text?.trim()) { + return jsonErrorResponse({ + status: 'error', + message: 'query_text cannot be empty', + }); + } + + if (metadata_filter) { + const err = validateMetadataFilter(metadata_filter); + if (err) return jsonErrorResponse({ status: 'error', message: err }); + } + + const flowCheck = requireSuggested(namespace); + if (!flowCheck.ok) { + return jsonErrorResponse({ status: 'error', message: flowCheck.message }); + } + + const chunkLimit = Math.min( + QUERY_DOCUMENTS_MAX_CHUNKS, + top_k * CHUNKS_PER_DOCUMENT + ); + const client = getPineconeClient(); + const results = await client.query({ + query: query_text.trim(), + topK: chunkLimit, + namespace, + useReranking: true, + metadataFilter: metadata_filter, + fields: undefined, + }); + + const reassembled = reassembleByDocument(results, { + maxChunksPerDocument: max_chunks_per_document ?? 200, + }); + + const topDocuments = reassembled + .sort((a, b) => b.best_score - a.best_score) + .slice(0, top_k); + + return jsonResponse({ + status: 'success', + query: query_text.trim(), + namespace, + metadata_filter: metadata_filter ?? undefined, + result_count: topDocuments.length, + documents: topDocuments.map((doc) => ({ + document_id: doc.document_id, + merged_content: doc.merged_content, + metadata: doc.metadata, + chunk_count: doc.chunk_count, + best_score: doc.best_score, + })), + }); + } catch (error) { + logToolError('query_documents', error); + return jsonErrorResponse({ + status: 'error', + message: getToolErrorMessage(error, 'Failed to query and reassemble documents'), + }); + } + } + ); +} From 41b3d99581a82f6f7cd4389779bdf742ef4841b5 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 00:07:41 +0800 Subject: [PATCH 07/20] #15- fix CI errors --- package-lock.json | 190 +++++++++++++---------- src/pinecone-client.ts | 7 +- src/server/format-query-result.ts | 3 +- src/server/metadata-filter.ts | 8 +- src/server/reassemble-documents.ts | 5 +- src/server/tools/query-documents-tool.ts | 5 +- 6 files changed, 125 insertions(+), 93 deletions(-) diff --git a/package-lock.json b/package-lock.json index c13df28..80b627f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -608,9 +608,9 @@ "license": "MIT" }, "node_modules/@eslint/js": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", - "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", + "version": "9.39.3", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.3.tgz", + "integrity": "sha512-1B1VkCq6FuUNlQvlBYb+1jDu/gV297TIs/OeiaSR9l1H27SVW55ONE1e1Vp16NqP683+xEGzxYtv4XCiDPaQiw==", "dev": true, "license": "MIT", "engines": { @@ -1159,22 +1159,23 @@ "integrity": "sha512-BkmoP5/FhRYek5izySdkOneRyXYN35I860MFAGupTdebyE66uZaR+bXLHq8k4DirE5DwQi3NuhvRU1jqTVwUrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.55.0.tgz", - "integrity": "sha512-1y/MVSz0NglV1ijHC8OT49mPJ4qhPYjiK08YUQVbIOyu+5k862LKUHFkpKHWu//zmr7hDR2rhwUm6gnCGNmGBQ==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.56.0.tgz", + "integrity": "sha512-lRyPDLzNCuae71A3t9NEINBiTn7swyOhvUj3MyUOxb8x6g6vPEFoOU+ZRmGMusNC3X3YMhqMIX7i8ShqhT74Pw==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/regexpp": "^4.12.2", - "@typescript-eslint/scope-manager": "8.55.0", - "@typescript-eslint/type-utils": "8.55.0", - "@typescript-eslint/utils": "8.55.0", - "@typescript-eslint/visitor-keys": "8.55.0", + "@typescript-eslint/scope-manager": "8.56.0", + "@typescript-eslint/type-utils": "8.56.0", + "@typescript-eslint/utils": "8.56.0", + "@typescript-eslint/visitor-keys": "8.56.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.4.0" @@ -1187,8 +1188,8 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "@typescript-eslint/parser": "^8.55.0", - "eslint": "^8.57.0 || ^9.0.0", + "@typescript-eslint/parser": "^8.56.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, @@ -1203,16 +1204,17 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.55.0.tgz", - "integrity": "sha512-4z2nCSBfVIMnbuu8uinj+f0o4qOeggYJLbjpPHka3KH1om7e+H9yLKTYgksTaHcGco+NClhhY2vyO3HsMH1RGw==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-8.56.0.tgz", + "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { - "@typescript-eslint/scope-manager": "8.55.0", - "@typescript-eslint/types": "8.55.0", - "@typescript-eslint/typescript-estree": "8.55.0", - "@typescript-eslint/visitor-keys": "8.55.0", + "@typescript-eslint/scope-manager": "8.56.0", + "@typescript-eslint/types": "8.56.0", + "@typescript-eslint/typescript-estree": "8.56.0", + "@typescript-eslint/visitor-keys": "8.56.0", "debug": "^4.4.3" }, "engines": { @@ -1223,19 +1225,19 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/project-service": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.55.0.tgz", - "integrity": "sha512-zRcVVPFUYWa3kNnjaZGXSu3xkKV1zXy8M4nO/pElzQhFweb7PPtluDLQtKArEOGmjXoRjnUZ29NjOiF0eCDkcQ==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/project-service/-/project-service-8.56.0.tgz", + "integrity": "sha512-M3rnyL1vIQOMeWxTWIW096/TtVP+8W3p/XnaFflhmcFp+U4zlxUxWj4XwNs6HbDeTtN4yun0GNTTDBw/SvufKg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/tsconfig-utils": "^8.55.0", - "@typescript-eslint/types": "^8.55.0", + "@typescript-eslint/tsconfig-utils": "^8.56.0", + "@typescript-eslint/types": "^8.56.0", "debug": "^4.4.3" }, "engines": { @@ -1250,14 +1252,14 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.55.0.tgz", - "integrity": "sha512-fVu5Omrd3jeqeQLiB9f1YsuK/iHFOwb04bCtY4BSCLgjNbOD33ZdV6KyEqplHr+IlpgT0QTZ/iJ+wT7hvTx49Q==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-8.56.0.tgz", + "integrity": "sha512-7UiO/XwMHquH+ZzfVCfUNkIXlp/yQjjnlYUyYz7pfvlK3/EyyN6BK+emDmGNyQLBtLGaYrTAI6KOw8tFucWL2w==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.55.0", - "@typescript-eslint/visitor-keys": "8.55.0" + "@typescript-eslint/types": "8.56.0", + "@typescript-eslint/visitor-keys": "8.56.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1268,9 +1270,9 @@ } }, "node_modules/@typescript-eslint/tsconfig-utils": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.55.0.tgz", - "integrity": "sha512-1R9cXqY7RQd7WuqSN47PK9EDpgFUK3VqdmbYrvWJZYDd0cavROGn+74ktWBlmJ13NXUQKlZ/iAEQHI/V0kKe0Q==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/tsconfig-utils/-/tsconfig-utils-8.56.0.tgz", + "integrity": "sha512-bSJoIIt4o3lKXD3xmDh9chZcjCz5Lk8xS7Rxn+6l5/pKrDpkCwtQNQQwZ2qRPk7TkUYhrq3WPIHXOXlbXP0itg==", "dev": true, "license": "MIT", "engines": { @@ -1285,15 +1287,15 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.55.0.tgz", - "integrity": "sha512-x1iH2unH4qAt6I37I2CGlsNs+B9WGxurP2uyZLRz6UJoZWDBx9cJL1xVN/FiOmHEONEg6RIufdvyT0TEYIgC5g==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-8.56.0.tgz", + "integrity": "sha512-qX2L3HWOU2nuDs6GzglBeuFXviDODreS58tLY/BALPC7iu3Fa+J7EOTwnX9PdNBxUI7Uh0ntP0YWGnxCkXzmfA==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.55.0", - "@typescript-eslint/typescript-estree": "8.55.0", - "@typescript-eslint/utils": "8.55.0", + "@typescript-eslint/types": "8.56.0", + "@typescript-eslint/typescript-estree": "8.56.0", + "@typescript-eslint/utils": "8.56.0", "debug": "^4.4.3", "ts-api-utils": "^2.4.0" }, @@ -1305,14 +1307,14 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/types": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.55.0.tgz", - "integrity": "sha512-ujT0Je8GI5BJWi+/mMoR0wxwVEQaxM+pi30xuMiJETlX80OPovb2p9E8ss87gnSVtYXtJoU9U1Cowcr6w2FE0w==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-8.56.0.tgz", + "integrity": "sha512-DBsLPs3GsWhX5HylbP9HNG15U0bnwut55Lx12bHB9MpXxQ+R5GC8MwQe+N1UFXxAeQDvEsEDY6ZYwX03K7Z6HQ==", "dev": true, "license": "MIT", "engines": { @@ -1324,16 +1326,16 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.55.0.tgz", - "integrity": "sha512-EwrH67bSWdx/3aRQhCoxDaHM+CrZjotc2UCCpEDVqfCE+7OjKAGWNY2HsCSTEVvWH2clYQK8pdeLp42EVs+xQw==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-8.56.0.tgz", + "integrity": "sha512-ex1nTUMWrseMltXUHmR2GAQ4d+WjkZCT4f+4bVsps8QEdh0vlBsaCokKTPlnqBFqqGaxilDNJG7b8dolW2m43Q==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/project-service": "8.55.0", - "@typescript-eslint/tsconfig-utils": "8.55.0", - "@typescript-eslint/types": "8.55.0", - "@typescript-eslint/visitor-keys": "8.55.0", + "@typescript-eslint/project-service": "8.56.0", + "@typescript-eslint/tsconfig-utils": "8.56.0", + "@typescript-eslint/types": "8.56.0", + "@typescript-eslint/visitor-keys": "8.56.0", "debug": "^4.4.3", "minimatch": "^9.0.5", "semver": "^7.7.3", @@ -1378,16 +1380,16 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.55.0.tgz", - "integrity": "sha512-BqZEsnPGdYpgyEIkDC1BadNY8oMwckftxBT+C8W0g1iKPdeqKZBtTfnvcq0nf60u7MkjFO8RBvpRGZBPw4L2ow==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-8.56.0.tgz", + "integrity": "sha512-RZ3Qsmi2nFGsS+n+kjLAYDPVlrzf7UhTffrDIKr+h2yzAlYP/y5ZulU0yeDEPItos2Ph46JAL5P/On3pe7kDIQ==", "dev": true, "license": "MIT", "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", - "@typescript-eslint/scope-manager": "8.55.0", - "@typescript-eslint/types": "8.55.0", - "@typescript-eslint/typescript-estree": "8.55.0" + "@typescript-eslint/scope-manager": "8.56.0", + "@typescript-eslint/types": "8.56.0", + "@typescript-eslint/typescript-estree": "8.56.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1397,19 +1399,19 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.55.0.tgz", - "integrity": "sha512-AxNRwEie8Nn4eFS1FzDMJWIISMGoXMb037sgCBJ3UR6o0fQTzr2tqN9WT+DkWJPhIdQCfV7T6D387566VtnCJA==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-8.56.0.tgz", + "integrity": "sha512-q+SL+b+05Ud6LbEE35qe4A99P+htKTKVbyiNEe45eCbJFyh/HVK9QXwlrbz+Q4L8SOW4roxSVwXYj4DMBT7Ieg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/types": "8.55.0", - "eslint-visitor-keys": "^4.2.1" + "@typescript-eslint/types": "8.56.0", + "eslint-visitor-keys": "^5.0.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -1419,6 +1421,19 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@typescript-eslint/visitor-keys/node_modules/eslint-visitor-keys": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-5.0.1.tgz", + "integrity": "sha512-tD40eHxA35h0PEIZNeIjkHoDR4YjjJp34biM0mDvplBe//mB+IHCqHDGV7pxF+7MklTvighcCPPZC7ynWyjdTA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": "^20.19.0 || ^22.13.0 || >=24" + }, + "funding": { + "url": "https://opencollective.com/eslint" + } + }, "node_modules/@vitest/expect": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", @@ -1549,6 +1564,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1567,9 +1583,9 @@ } }, "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -2020,11 +2036,12 @@ } }, "node_modules/eslint": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", - "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "version": "9.39.3", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.3.tgz", + "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -2032,7 +2049,7 @@ "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.2", + "@eslint/js": "9.39.3", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -2268,6 +2285,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -2615,10 +2633,11 @@ } }, "node_modules/hono": { - "version": "4.11.6", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.6.tgz", - "integrity": "sha512-ofIiiHyl34SV6AuhE3YT2mhO5HRWokce+eUYE82TsP6z0/H3JeJcjVWEMSIAiw2QkjDOEpES/lYsg8eEbsLtdw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.0.tgz", + "integrity": "sha512-NekXntS5M94pUfiVZ8oXXK/kkri+5WpX2/Ik+LVsl+uvw+soj4roXIsPqO+XsWrAw20mOzaXOZf3Q7PfB9A/IA==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -3143,6 +3162,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -3238,9 +3258,9 @@ } }, "node_modules/qs": { - "version": "6.14.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", - "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", + "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -3667,6 +3687,7 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -3714,6 +3735,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -3723,16 +3745,16 @@ } }, "node_modules/typescript-eslint": { - "version": "8.55.0", - "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.55.0.tgz", - "integrity": "sha512-HE4wj+r5lmDVS9gdaN0/+iqNvPZwGfnJ5lZuz7s5vLlg9ODw0bIiiETaios9LvFI1U94/VBXGm3CB2Y5cNFMpw==", + "version": "8.56.0", + "resolved": "https://registry.npmjs.org/typescript-eslint/-/typescript-eslint-8.56.0.tgz", + "integrity": "sha512-c7toRLrotJ9oixgdW7liukZpsnq5CZ7PuKztubGYlNppuTqhIoWfhgHo/7EU0v06gS2l/x0i2NEFK1qMIf0rIg==", "dev": true, "license": "MIT", "dependencies": { - "@typescript-eslint/eslint-plugin": "8.55.0", - "@typescript-eslint/parser": "8.55.0", - "@typescript-eslint/typescript-estree": "8.55.0", - "@typescript-eslint/utils": "8.55.0" + "@typescript-eslint/eslint-plugin": "8.56.0", + "@typescript-eslint/parser": "8.56.0", + "@typescript-eslint/typescript-estree": "8.56.0", + "@typescript-eslint/utils": "8.56.0" }, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3742,7 +3764,7 @@ "url": "https://opencollective.com/typescript-eslint" }, "peerDependencies": { - "eslint": "^8.57.0 || ^9.0.0", + "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.0.0" } }, @@ -3787,6 +3809,7 @@ "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -4000,6 +4023,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 385c72e..0cf4f1f 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -7,7 +7,12 @@ */ import { Pinecone } from '@pinecone-database/pinecone'; -import { debug as logDebug, error as logError, info as logInfo, warn as logWarn } from './logger.js'; +import { + debug as logDebug, + error as logError, + info as logInfo, + warn as logWarn, +} from './logger.js'; import type { PineconeClientConfig, SearchResult, diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts index 0c6f72c..9543253 100644 --- a/src/server/format-query-result.ts +++ b/src/server/format-query-result.ts @@ -37,8 +37,7 @@ export function formatSearchResultAsRow( if (options?.enrichUrls && options?.namespace) { const generated = generateUrlForNamespace(options.namespace, metadata); const existingUrl = metadata['url']; - const urlIsBlank = - typeof existingUrl !== 'string' || existingUrl.trim() === ''; + const urlIsBlank = typeof existingUrl !== 'string' || existingUrl.trim() === ''; if (generated.url && urlIsBlank) { metadata['url'] = generated.url; } diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index a02c66b..b9be397 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -55,8 +55,12 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; } if ( - (key === '$eq' || key === '$ne' || key === '$gt' || key === '$gte' || - key === '$lt' || key === '$lte') && + (key === '$eq' || + key === '$ne' || + key === '$gt' || + key === '$gte' || + key === '$lt' || + key === '$lte') && !isPrimitiveFilterValue(nestedValue) ) { return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; diff --git a/src/server/reassemble-documents.ts b/src/server/reassemble-documents.ts index f11522a..99c4232 100644 --- a/src/server/reassemble-documents.ts +++ b/src/server/reassemble-documents.ts @@ -87,7 +87,10 @@ export function reassembleByDocument( }); const toMerge = sorted.slice(0, maxChunks); - const merged_content = toMerge.map((c) => c.content.trim()).filter(Boolean).join(separator); + const merged_content = toMerge + .map((c) => c.content.trim()) + .filter(Boolean) + .join(separator); const first = toMerge[0]; const best_score = Math.max(...toMerge.map((c) => c.score)); diff --git a/src/server/tools/query-documents-tool.ts b/src/server/tools/query-documents-tool.ts index 896e256..6416e5f 100644 --- a/src/server/tools/query-documents-tool.ts +++ b/src/server/tools/query-documents-tool.ts @@ -81,10 +81,7 @@ export function registerQueryDocumentsTool(server: McpServer): void { return jsonErrorResponse({ status: 'error', message: flowCheck.message }); } - const chunkLimit = Math.min( - QUERY_DOCUMENTS_MAX_CHUNKS, - top_k * CHUNKS_PER_DOCUMENT - ); + const chunkLimit = Math.min(QUERY_DOCUMENTS_MAX_CHUNKS, top_k * CHUNKS_PER_DOCUMENT); const client = getPineconeClient(); const results = await client.query({ query: query_text.trim(), From ee783e00e34002200827cfa3fc67eb6f8edd67c1 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 01:38:16 +0800 Subject: [PATCH 08/20] #21-address remaining CodeRabbit threads: fix CHANGELOG [Unreleased] and COUNT_TOP_K JSDoc Co-authored-by: Cursor --- CHANGELOG.md | 37 +++++++++++++++++-------------------- src/constants.ts | 14 ++++++++++++-- 2 files changed, 29 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 369b30f..37f4a3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,32 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Initial TypeScript implementation of Pinecone Read-Only MCP -- Hybrid search support (dense + sparse embeddings) -- Semantic reranking using BGE reranker model -- Dynamic namespace discovery -- Metadata filtering support -- Full TypeScript type definitions -- Comprehensive test suite -- GitHub Actions CI/CD workflows -- ESLint and Prettier configuration -- Complete documentation +- `list_namespaces` tool: discovers all namespaces with record counts and sampled metadata fields +- `namespace_router` tool: ranks namespaces by relevance to a natural-language query +- `suggest_query_params` tool: mandatory flow gate that recommends fields and tool variant before queries +- `count` tool: counts unique documents matching a query, with `truncated` flag when results exceed `COUNT_TOP_K` +- `query_fast` tool: lightweight chunk-level search with minimal field set +- `query_detailed` tool: chunk-level search with optional semantic reranking +- `query` tool: unified query entry-point supporting `query_fast` and `query_detailed` modes +- `guided_query` tool: single-call orchestrator that runs routing → suggestion → execution, returning a `decision_trace` +- `generate_urls` tool: synthesizes URLs for records whose metadata lacks a `url` field +- `query_documents` tool: reassembles full documents from chunks grouped by document identifier +- Centralized structured logger (`src/logger.ts`) with level-gated stderr output and stack-trace capture +- Dockerfile for containerised deployment +- `About.md` project documentation covering architecture, tools, and operating principles ### Changed -- N/A - -### Deprecated - -- N/A - -### Removed - -- N/A +- Modularised server into focused files under `src/server/` (tools, caches, formatters, suggestion flow) +- CI workflow updated: multi-node matrix (`18.x`, `20.x`, `22.x`), separate quality job with `continue-on-error` +- Replaced all `console.error` calls in `pinecone-client.ts` with typed `logInfo` / `logDebug` / `logError` ### Fixed -- N/A +- Timestamp-based metadata filter now correctly handles numeric epoch values ### Security diff --git a/src/constants.ts b/src/constants.ts index 397cf41..0aa7ade 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -9,9 +9,19 @@ export const MAX_TOP_K = 100; export const MIN_TOP_K = 1; /** Namespace and suggestion caches stay valid for 30 minutes. */ export const FLOW_CACHE_TTL_MS = 30 * 60 * 1000; -/** Top-k used by the count tool to get total matching documents (Pinecone allows high values). */ +/** + * Maximum hits fetched by the count tool to deduplicate into a document count. + * When the matching set exceeds this limit the count is capped; callers should + * check the `truncated: true` flag in the response to detect this condition. + */ export const COUNT_TOP_K = 10_000; -/** Minimal fields requested for count (no chunk_text) to reduce payload and cost. */ +/** + * Minimal fields fetched for count queries (no `chunk_text`) to reduce payload and cost. + * All three fields are tried as deduplication keys in priority order: + * 1. `document_number` — canonical document identifier used by most namespaces + * 2. `url` — used as a fallback document key when document_number is absent + * 3. `doc_id` — secondary fallback for namespaces that use a doc_id scheme + */ export const COUNT_FIELDS = ['document_number', 'url', 'doc_id'] as const; /** Default lightweight field set for fast queries. */ export const FAST_QUERY_FIELDS = ['document_number', 'title', 'url', 'author', 'doc_id'] as const; From cebf33bfb5ae8a1ce496157312b0cb040c23b627 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 02:59:16 +0800 Subject: [PATCH 09/20] #21-fix: expand CHUNKS_PER_DOCUMENT comment, clarify rerank cast, fix fields nullable Co-authored-by: Cursor --- src/pinecone-client.ts | 14 ++++++++++++++ src/server/tools/guided-query-tool.ts | 8 +++++--- src/server/tools/query-documents-tool.ts | 8 +++++++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 0cf4f1f..9310bf7 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -332,6 +332,12 @@ export class PineconeClient { const rerankResult = await pc.inference.rerank( this.rerankModel, query, + // The Pinecone SDK types constrain document values to `Record`, + // but the underlying HTTP API accepts any JSON value. We pass MergedHit objects + // (metadata may contain number/boolean/string[]) and only `chunk_text` — which is + // always a string — is accessed via rankFields. The double cast via `as unknown` + // is intentional: it bypasses the SDK's over-narrow type without stringifying + // metadata values that we need to read back from the returned documents. results as unknown as (string | Record)[], { topN, @@ -456,6 +462,9 @@ export class PineconeClient { * to avoid transferring chunk content, and deduplicates by document for a document-level count. */ async count(params: CountParams): Promise { + if (!params.query || !params.query.trim()) { + throw new Error('Query cannot be empty'); + } const { denseIndex } = await this.ensureIndexes(); const hits = await this.searchIndex( @@ -485,6 +494,11 @@ export class PineconeClient { idFallbackCount++; docKeys.add(hit._id ?? ''); } + if (!docNumber && !url && !docId) { + logWarn( + `count(): hit ${hit._id} in namespace "${params.namespace}" had none of the identifier fields (${COUNT_FIELDS.join(', ')})` + ); + } } if (idFallbackCount > 0) { logWarn( diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts index 4a17b9d..bebbb0a 100644 --- a/src/server/tools/guided-query-tool.ts +++ b/src/server/tools/guided-query-tool.ts @@ -141,14 +141,16 @@ export function registerGuidedQueryTool(server: McpServer): void { const fields = suggestion.suggested_fields.length > 0 ? suggestion.suggested_fields - : [...FAST_QUERY_FIELDS]; + : isFast + ? [...FAST_QUERY_FIELDS] + : undefined; const queryResults = await client.query({ query: queryText, namespace, topK: top_k, metadataFilter: metadata_filter, useReranking: !isFast, - fields, + fields: fields?.length ? fields : undefined, }); const formattedResults = formatQueryResultRows(queryResults, { namespace, @@ -161,7 +163,7 @@ export function registerGuidedQueryTool(server: McpServer): void { namespace, metadata_filter: metadata_filter, result_count: formattedResults.length, - ...(fields.length > 0 ? { fields } : {}), + ...(fields?.length ? { fields } : {}), results: formattedResults, }; return jsonResponse({ diff --git a/src/server/tools/query-documents-tool.ts b/src/server/tools/query-documents-tool.ts index 6416e5f..7dbb285 100644 --- a/src/server/tools/query-documents-tool.ts +++ b/src/server/tools/query-documents-tool.ts @@ -12,7 +12,13 @@ import { requireSuggested } from '../suggestion-flow.js'; import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; -/** Approximate chunks to fetch per requested document (to get enough for reassembly). */ +/** + * Heuristic multiplier: chunks fetched = top_k × CHUNKS_PER_DOCUMENT, capped by + * QUERY_DOCUMENTS_MAX_CHUNKS. Set to 50 as a balance between recall and performance — + * documents with more than ~50 chunks may be truncated unless the caller passes a + * higher `max_chunks_per_document` (default 200, max 500). Increasing this constant + * raises Pinecone fetch latency and memory usage during reassembly. + */ const CHUNKS_PER_DOCUMENT = 50; export function registerQueryDocumentsTool(server: McpServer): void { From 7da74f629f479586df590f7d5001161f7c2791bd Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 03:02:13 +0800 Subject: [PATCH 10/20] #15-process some exception --- src/server/query-suggestion.ts | 12 +++++++++++- src/server/reassemble-documents.ts | 8 ++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/server/query-suggestion.ts b/src/server/query-suggestion.ts index 925de66..f4a62d3 100644 --- a/src/server/query-suggestion.ts +++ b/src/server/query-suggestion.ts @@ -21,7 +21,7 @@ export function suggestQueryParams( const available = namespaceMetadataFields ? Object.keys(namespaceMetadataFields) : []; const keepOnlyAvailable = (fields: string[]) => fields.filter((f) => available.includes(f)); - if (!namespaceMetadataFields || available.length === 0) { + if (!namespaceMetadataFields) { return { suggested_fields: [], use_count_tool: false, @@ -31,6 +31,16 @@ export function suggestQueryParams( namespace_found: false, }; } + if (available.length === 0) { + return { + suggested_fields: [], + use_count_tool: false, + recommended_tool: 'query_fast', + explanation: + 'Namespace has no metadata fields. Use list_namespaces to verify the namespace is correct.', + namespace_found: true, + }; + } // Count intent: "how many", "count", "number of", etc. if (/\b(how many|count|number of|total number|paper count|documents? count)\b/.test(q)) { diff --git a/src/server/reassemble-documents.ts b/src/server/reassemble-documents.ts index 99c4232..83b66db 100644 --- a/src/server/reassemble-documents.ts +++ b/src/server/reassemble-documents.ts @@ -7,7 +7,7 @@ import type { PineconeMetadataValue, SearchResult } from '../types.js'; /** Default metadata keys tried for chunk ordering (RecursiveCharacterTextSplitter often adds these). */ -const CHUNK_ORDER_KEYS = ['chunk_index', 'chunk_index_0', 'index', 'loc'] as const; +const CHUNK_ORDER_KEYS = ['chunk_index', 'start_index', 'loc'] as const; function getDocumentKey(hit: SearchResult): string { const m = hit.metadata || {}; @@ -48,6 +48,10 @@ export interface ReassembledDocument { /** * Group search results by document and merge chunk content. * Chunks are ordered by metadata chunk_index (or similar) when present; otherwise retrieval order. + * * + * Document identity is derived in priority order: document_number → url → doc_id → hit.id. + * Chunks whose metadata contains none of the first three keys are grouped by their raw vector ID, + * so each such chunk becomes its own single-chunk "document" in the output. */ export function reassembleByDocument( results: SearchResult[], @@ -58,7 +62,7 @@ export function reassembleByDocument( contentSeparator?: string; } ): ReassembledDocument[] { - const maxChunks = options?.maxChunksPerDocument ?? 200; + const maxChunks = Math.max(1, options?.maxChunksPerDocument ?? 200); const separator = options?.contentSeparator ?? '\n\n'; const byDoc = new Map(); From 3d98105dc8587419f0188d07d2280cb2c7767308 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 03:16:32 +0800 Subject: [PATCH 11/20] #15-remove unnecessary condition statement --- src/pinecone-client.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 9310bf7..c74f351 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -494,11 +494,6 @@ export class PineconeClient { idFallbackCount++; docKeys.add(hit._id ?? ''); } - if (!docNumber && !url && !docId) { - logWarn( - `count(): hit ${hit._id} in namespace "${params.namespace}" had none of the identifier fields (${COUNT_FIELDS.join(', ')})` - ); - } } if (idFallbackCount > 0) { logWarn( From 52ba00de0ad3cc70e3ce8b053adb66f4a0fa24c6 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 03:42:16 +0800 Subject: [PATCH 12/20] #15-update docstring --- src/index.ts | 3 +++ src/logger.ts | 8 ++++++++ src/pinecone-client.ts | 1 + src/server.ts | 1 + src/server/client-context.ts | 2 ++ src/server/metadata-filter.ts | 4 ++++ src/server/namespace-router.ts | 4 ++++ src/server/namespaces-cache.ts | 2 ++ src/server/reassemble-documents.ts | 2 ++ src/server/suggestion-flow.ts | 2 ++ src/server/tool-response.ts | 2 ++ src/server/tools/count-tool.ts | 1 + src/server/tools/generate-urls-tool.ts | 2 ++ src/server/tools/guided-query-tool.ts | 1 + src/server/tools/list-namespaces-tool.ts | 1 + src/server/tools/namespace-router-tool.ts | 1 + src/server/tools/query-documents-tool.ts | 1 + src/server/tools/query-tool.ts | 2 ++ src/server/tools/suggest-query-params-tool.ts | 1 + src/server/url-generation.ts | 3 +++ 20 files changed, 44 insertions(+) diff --git a/src/index.ts b/src/index.ts index 0c9b225..67f9bfc 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,6 +26,7 @@ interface CLIOptions { logLevel?: string; } +/** Parse argv into API key, index name, rerank model, and log level. */ function parseArgs(): CLIOptions { const args = process.argv.slice(2); const options: CLIOptions = {}; @@ -62,6 +63,7 @@ function parseArgs(): CLIOptions { return options; } +/** Print CLI usage and exit. */ function printHelp(): void { console.log(` Pinecone Read-Only MCP Server @@ -94,6 +96,7 @@ Examples: `); } +/** Initialize config, Pinecone client, MCP server, and stdio transport. */ async function main(): Promise { try { const options = parseArgs(); diff --git a/src/logger.ts b/src/logger.ts index 0e940c0..a8e470a 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -14,18 +14,22 @@ const LEVEL_ORDER: Record = { let currentLevel: LogLevel = 'INFO'; +/** Set the minimum log level (DEBUG, INFO, WARN, ERROR). Messages below this level are dropped. */ export function setLogLevel(level: LogLevel): void { currentLevel = level; } +/** Return the current minimum log level. */ export function getLogLevel(): LogLevel { return currentLevel; } +/** True if the given level is at or above the current minimum. */ function shouldLog(level: LogLevel): boolean { return LEVEL_ORDER[level] >= LEVEL_ORDER[currentLevel]; } +/** Format a single log line: timestamp, level, message, and optional JSON data. */ function formatMessage(level: string, msg: string, data?: unknown): string { const ts = new Date().toISOString(); const prefix = `[${ts}] [${level}]`; @@ -35,24 +39,28 @@ function formatMessage(level: string, msg: string, data?: unknown): string { return `${prefix} ${msg}`; } +/** Log a DEBUG-level message to stderr when the log level allows. */ export function debug(msg: string, data?: unknown): void { if (shouldLog('DEBUG')) { console.error(formatMessage('DEBUG', msg, data)); } } +/** Log an INFO-level message to stderr when the log level allows. */ export function info(msg: string, data?: unknown): void { if (shouldLog('INFO')) { console.error(formatMessage('INFO', msg, data)); } } +/** Log a WARN-level message to stderr when the log level allows. */ export function warn(msg: string, data?: unknown): void { if (shouldLog('WARN')) { console.error(formatMessage('WARN', msg, data)); } } +/** Log an ERROR-level message to stderr with optional error (message and stack). */ export function error(msg: string, err?: unknown): void { if (shouldLog('ERROR')) { // const detail = err instanceof Error ? err.message : err !== undefined ? String(err) : undefined; diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index c74f351..ce6d86b 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -64,6 +64,7 @@ export class PineconeClient { private sparseIndex: SearchableIndex | null = null; private initialized = false; + /** Create a client with the given config; env vars override index name, rerank model, and top-k. */ constructor(config: PineconeClientConfig) { this.apiKey = config.apiKey; this.indexName = config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; diff --git a/src/server.ts b/src/server.ts index 61e0d7f..a9beb82 100644 --- a/src/server.ts +++ b/src/server.ts @@ -21,6 +21,7 @@ export { validateMetadataFilter } from './server/metadata-filter.js'; export { suggestQueryParams } from './server/query-suggestion.js'; export type { SuggestQueryParamsResult } from './server/query-suggestion.js'; +/** Create and configure the MCP server with all tools; returns the server instance. */ export async function setupServer(): Promise { const server = new McpServer( { diff --git a/src/server/client-context.ts b/src/server/client-context.ts index 9850ba7..f2f759f 100644 --- a/src/server/client-context.ts +++ b/src/server/client-context.ts @@ -3,6 +3,7 @@ import { PineconeClient } from '../pinecone-client.js'; // Global Pinecone client (initialized lazily) let pineconeClient: PineconeClient | null = null; +/** Return the shared Pinecone client; throws if setPineconeClient has not been called. */ export function getPineconeClient(): PineconeClient { if (!pineconeClient) { throw new Error('Pinecone client not initialized. Call setPineconeClient first.'); @@ -10,6 +11,7 @@ export function getPineconeClient(): PineconeClient { return pineconeClient; } +/** Set the shared Pinecone client used by all MCP tools. */ export function setPineconeClient(client: PineconeClient): void { pineconeClient = client; } diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index b9be397..a98c76a 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -25,14 +25,17 @@ const ALLOWED_FILTER_OPERATORS = new Set([ '$nin', ]); +/** True if value is a string, number, or boolean (allowed for $eq, $gt, etc.). */ function isPrimitiveFilterValue(value: unknown): boolean { return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; } +/** True if value is an array of primitives (required for $in/$nin). */ function isPrimitiveArray(value: unknown): boolean { return Array.isArray(value) && value.every((item) => isPrimitiveFilterValue(item)); } +/** Recursively validate a filter value; returns an error string or null if valid. */ function validateMetadataFilterValue(value: unknown, path: string[]): string | null { if (value === null || value === undefined) { return `Invalid null/undefined at "${path.join('.')}".`; @@ -76,6 +79,7 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n return null; } +/** Validate a Pinecone metadata filter object; returns an error message or null if valid. */ export function validateMetadataFilter(filter: Record): string | null { for (const [field, value] of Object.entries(filter)) { const error = validateMetadataFilterValue(value, [field]); diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts index 72baf35..60e6ef6 100644 --- a/src/server/namespace-router.ts +++ b/src/server/namespace-router.ts @@ -47,6 +47,10 @@ function scoreNamespace( return { score, reasons: Array.from(new Set(reasons)) }; } +/** + * Rank namespaces by relevance to the query and return the top N. + * Uses name and metadata-field matching; on equal score, prefers smaller (more-specific) namespaces. + */ export function rankNamespacesByQuery( query: string, namespaces: NamespaceInfo[], diff --git a/src/server/namespaces-cache.ts b/src/server/namespaces-cache.ts index bd51579..a40fe1c 100644 --- a/src/server/namespaces-cache.ts +++ b/src/server/namespaces-cache.ts @@ -14,6 +14,7 @@ type CacheEntry = { let namespacesCache: CacheEntry | null = null; +/** Return namespace list with metadata; uses in-memory cache for FLOW_CACHE_TTL_MS. */ export async function getNamespacesWithCache(): Promise<{ data: NamespaceInfo[]; cache_hit: boolean; @@ -35,6 +36,7 @@ export async function getNamespacesWithCache(): Promise<{ return { data, cache_hit: false, expires_at: expiresAt }; } +/** Clear the namespaces cache so the next call to getNamespacesWithCache refetches. */ export function invalidateNamespacesCache(): void { namespacesCache = null; } diff --git a/src/server/reassemble-documents.ts b/src/server/reassemble-documents.ts index 83b66db..c66590e 100644 --- a/src/server/reassemble-documents.ts +++ b/src/server/reassemble-documents.ts @@ -9,6 +9,7 @@ import type { PineconeMetadataValue, SearchResult } from '../types.js'; /** Default metadata keys tried for chunk ordering (RecursiveCharacterTextSplitter often adds these). */ const CHUNK_ORDER_KEYS = ['chunk_index', 'start_index', 'loc'] as const; +/** Derive a stable document key from hit metadata: document_number → url → doc_id → hit.id. */ function getDocumentKey(hit: SearchResult): string { const m = hit.metadata || {}; const docNumber = m['document_number']; @@ -23,6 +24,7 @@ function getDocumentKey(hit: SearchResult): string { ); } +/** Return a numeric order for sorting chunks from metadata (chunk_index, start_index, or loc). */ function getChunkOrder(metadata: Record): number { for (const key of CHUNK_ORDER_KEYS) { const v = metadata[key]; diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts index ecf12f5..de67480 100644 --- a/src/server/suggestion-flow.ts +++ b/src/server/suggestion-flow.ts @@ -22,6 +22,7 @@ function sweepExpired(): void { } } +/** Record that suggest_query_params was called for this namespace (enables query/count for the flow). */ export function markSuggested(namespace: string, state: Omit): void { sweepExpired(); stateByNamespace.set(namespace, { @@ -30,6 +31,7 @@ export function markSuggested(namespace: string, state: Omit): Record { const nested = record['metadata']; if (nested && typeof nested === 'object' && !Array.isArray(nested)) { @@ -12,6 +13,7 @@ function extractMetadata(record: Record): Record) => UrlGenerationResult; /** Registry of namespace -> URL generator. Built-ins are registered below; more can be added at runtime. */ const urlGenerators = new Map(); +/** Return a trimmed non-empty string or null for empty/missing values. */ function asString(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; } +/** Build a mailing-list URL from doc_id or thread_id (e.g. Boost archives). */ function generatorMailing(metadata: Record): UrlGenerationResult { const docIdOrThread = asString(metadata['doc_id']) ?? asString(metadata['thread_id']); if (!docIdOrThread) { @@ -33,6 +35,7 @@ function generatorMailing(metadata: Record): UrlGenerationResul }; } +/** Build a Slack message URL from source or team_id/channel_id/doc_id. */ function generatorSlackCpplang(metadata: Record): UrlGenerationResult { const source = asString(metadata['source']); if (source) { From 5c74f7104736aa7bdb38d4d8690a208c9bfc2060 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 04:13:34 +0800 Subject: [PATCH 13/20] #15-fix validation logic for array type --- src/server/metadata-filter.ts | 42 ++++++++++++++++++---------------- src/server/namespace-router.ts | 3 ++- src/server/url-generation.ts | 2 +- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index a98c76a..196301f 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -30,9 +30,9 @@ function isPrimitiveFilterValue(value: unknown): boolean { return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; } -/** True if value is an array of primitives (required for $in/$nin). */ +/** True if value is an array of string (required for $in/$nin). */ function isPrimitiveArray(value: unknown): boolean { - return Array.isArray(value) && value.every((item) => isPrimitiveFilterValue(item)); + return Array.isArray(value) && value.every((item) => typeof item === 'string'); } /** Recursively validate a filter value; returns an error string or null if valid. */ @@ -50,25 +50,27 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n } for (const [key, nestedValue] of Object.entries(value)) { - if (key.startsWith('$')) { - if (!ALLOWED_FILTER_OPERATORS.has(key)) { - return `Unsupported filter operator "${key}" at "${path.join('.')}".`; - } - if ((key === '$in' || key === '$nin') && !isPrimitiveArray(nestedValue)) { - return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; - } - if ( - (key === '$eq' || - key === '$ne' || - key === '$gt' || - key === '$gte' || - key === '$lt' || - key === '$lte') && - !isPrimitiveFilterValue(nestedValue) - ) { - return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; - } + if (!key.startsWith('$')) { + return `Unsupported filter operator "${key}" at "${path.join('.')}".`; } + if (!ALLOWED_FILTER_OPERATORS.has(key)) { + return `Unsupported filter operator "${key}" at "${path.join('.')}".`; + } + if ((key === '$in' || key === '$nin') && !isPrimitiveArray(nestedValue)) { + return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; + } + if ( + (key === '$eq' || + key === '$ne' || + key === '$gt' || + key === '$gte' || + key === '$lt' || + key === '$lte') && + !isPrimitiveFilterValue(nestedValue) + ) { + return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; + } + const nestedError = validateMetadataFilterValue(nestedValue, [...path, key]); if (nestedError) { diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts index 60e6ef6..2f37cb5 100644 --- a/src/server/namespace-router.ts +++ b/src/server/namespace-router.ts @@ -56,6 +56,7 @@ export function rankNamespacesByQuery( namespaces: NamespaceInfo[], topN: number ): RankedNamespace[] { + const limit = Math.max(1, Math.floor(topN)); return namespaces .map((ns) => { const fields = Object.keys(ns.metadata ?? {}); @@ -73,5 +74,5 @@ export function rankNamespacesByQuery( // targeted namespaces are chosen over large catch-all ones. return a.record_count - b.record_count; }) - .slice(0, topN); + .slice(0, limit); } diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index e47953a..14b3755 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -30,7 +30,7 @@ function generatorMailing(metadata: Record): UrlGenerationResul }; } return { - url: `https://lists.boost.org/archives/list/${docIdOrThread}/`, + url: `https://lists.boost.org/archives/list/${encodeURIComponent(docIdOrThread)}/`, method: 'generated.mailing', }; } From 1a014e6105bbe62095beb4f15ccbd178342d1d54 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 04:20:40 +0800 Subject: [PATCH 14/20] #15-fix ci format error --- src/server/metadata-filter.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index 196301f..921ee05 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -71,7 +71,6 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; } - const nestedError = validateMetadataFilterValue(nestedValue, [...path, key]); if (nestedError) { return nestedError; From 6c5269d7a6a9217e14086a39310943e6d6c5e133 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 04:32:12 +0800 Subject: [PATCH 15/20] #15-removed encodeURIComponent --- src/server/url-generation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index 14b3755..e47953a 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -30,7 +30,7 @@ function generatorMailing(metadata: Record): UrlGenerationResul }; } return { - url: `https://lists.boost.org/archives/list/${encodeURIComponent(docIdOrThread)}/`, + url: `https://lists.boost.org/archives/list/${docIdOrThread}/`, method: 'generated.mailing', }; } From 029c07be25351b18ee66ff291170098e433fbfe3 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 05:11:37 +0800 Subject: [PATCH 16/20] #15-fix to process and, or operator --- src/server/metadata-filter.ts | 18 +++++++++++++++++- src/server/namespace-router.ts | 6 +++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index 921ee05..98fb3a2 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -9,6 +9,7 @@ const metadataFilterValueSchema: z.ZodType = z.lazy(() => z.boolean(), z.array(z.string()), z.array(z.number()), + z.array(z.lazy(() => metadataFilterSchema)), z.record(z.string(), metadataFilterValueSchema), // Recursive for nested operators ]) ); @@ -23,6 +24,8 @@ const ALLOWED_FILTER_OPERATORS = new Set([ '$lte', '$in', '$nin', + '$and', + '$or', ]); /** True if value is a string, number, or boolean (allowed for $eq, $gt, etc.). */ @@ -45,7 +48,17 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n return null; } - if (typeof value !== 'object' || Array.isArray(value)) { + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + const parsed = metadataFilterSchema.safeParse(value[i]); + if (!parsed.success) { + return `Operator "${path[path.length - 1]}" at "${path.join('.')}" must use an array of filter objects.`; + } + } + return null; + } + + if (typeof value !== 'object') { return `Unsupported filter value at "${path.join('.')}".`; } @@ -70,6 +83,9 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n ) { return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; } + if ((key === '$and' || key === '$or') && !Array.isArray(nestedValue)) { + return `Operator "${key}" at "${path.join('.')}" must use an array of filter objects.`; + } const nestedError = validateMetadataFilterValue(nestedValue, [...path, key]); if (nestedError) { diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts index 2f37cb5..5db484a 100644 --- a/src/server/namespace-router.ts +++ b/src/server/namespace-router.ts @@ -1,6 +1,6 @@ import type { NamespaceInfo } from './namespaces-cache.js'; -type RankedNamespace = { +export type RankedNamespace = { namespace: string; score: number; record_count: number; @@ -28,7 +28,7 @@ function scoreNamespace( score += 3; reasons.push('query mentions namespace name'); } else { - const nameTokens = normalizedName.split(/\s+/).filter(Boolean); + const nameTokens = [...new Set(normalizedName.split(/\s+/).filter(Boolean))]; for (const token of nameTokens) { if (token.length >= 2 && q.includes(token)) { score += 2; @@ -56,7 +56,7 @@ export function rankNamespacesByQuery( namespaces: NamespaceInfo[], topN: number ): RankedNamespace[] { - const limit = Math.max(1, Math.floor(topN)); + const limit = Number.isFinite(topN) ? Math.max(1, Math.floor(topN)) : 1; return namespaces .map((ns) => { const fields = Object.keys(ns.metadata ?? {}); From 59aaef3bb6ce777b5c0f2ad69a944a655c05316e Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 12:41:01 +0800 Subject: [PATCH 17/20] #15-fix second errors --- package.json | 4 ++-- src/index.ts | 2 -- src/logger.ts | 14 ++++++++------ src/pinecone-client.ts | 6 ++---- src/server/format-query-result.ts | 6 ++++-- src/server/tools/count-tool.ts | 4 ++-- src/server/tools/generate-urls-tool.ts | 1 + src/server/tools/query-documents-tool.ts | 2 +- src/server/tools/query-tool.ts | 8 ++++---- 9 files changed, 24 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 1b3e8f8..de5250e 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,7 @@ }, "scripts": { "clean": "node -e \"const fs=require('fs'); fs.rmSync('dist',{recursive:true,force:true});\"", - "build": "npm run clean && node ./node_modules/typescript/bin/tsc", + "build": "npm run clean && npx tsc", "build:watch": "tsc --watch", "dev": "tsx watch src/index.ts", "start": "node dist/index.js", @@ -52,7 +52,7 @@ "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "typecheck": "tsc --noEmit", "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm test", - "release:check": "npm run ci && npm pack --dry-run", + "release:check": "npm run ci && npm pack --dry-run --ignore-scripts", "ci:local": "bash scripts/ci-local.sh", "prepublishOnly": "npm run ci", "prepack": "npm run build" diff --git a/src/index.ts b/src/index.ts index 67f9bfc..bf28d98 100644 --- a/src/index.ts +++ b/src/index.ts @@ -105,12 +105,10 @@ async function main(): Promise { const rawLevel = options.logLevel || process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || - process.env['LOG_LEVEL'] || 'INFO'; const logLevel = ( ['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) ? rawLevel : 'INFO' ) as LogLevel; - process.env['LOG_LEVEL'] = logLevel; setLogLevel(logLevel); // Get API key diff --git a/src/logger.ts b/src/logger.ts index a8e470a..8dc5392 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -30,11 +30,17 @@ function shouldLog(level: LogLevel): boolean { } /** Format a single log line: timestamp, level, message, and optional JSON data. */ -function formatMessage(level: string, msg: string, data?: unknown): string { +function formatMessage(level: LogLevel, msg: string, data?: unknown): string { const ts = new Date().toISOString(); const prefix = `[${ts}] [${level}]`; if (data !== undefined) { - return `${prefix} ${msg} ${JSON.stringify(data)}`; + let serialized: string; + try { + serialized = JSON.stringify(data); + } catch { + serialized = String(data); + } + return `${prefix} ${msg} ${serialized}`; } return `${prefix} ${msg}`; } @@ -63,10 +69,6 @@ export function warn(msg: string, data?: unknown): void { /** Log an ERROR-level message to stderr with optional error (message and stack). */ export function error(msg: string, err?: unknown): void { if (shouldLog('ERROR')) { - // const detail = err instanceof Error ? err.message : err !== undefined ? String(err) : undefined; - // console.error( - // formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined) - // ); const detail = err instanceof Error ? { message: err.message, stack: err.stack } diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index ce6d86b..64a4295 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -168,7 +168,7 @@ export class PineconeClient { if (!(key in metadataFields)) { metadataFields[key] = inferredType; } else if ( - metadataFields[key] === 'object' && + metadataFields[key] === 'object' || metadataFields[key] === 'array' && inferredType === 'string[]' ) { // Prefer array type over generic object when we see it in another sample @@ -227,9 +227,7 @@ export class PineconeClient { // Include filter when explicitly provided (matches Python behavior). if (metadataFilter !== undefined) { queryPayload['filter'] = metadataFilter; - if (!options?.fields) { - logDebug('Applying metadata filter', metadataFilter); - } + logDebug('Applying metadata filter', metadataFilter); } try { diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts index 9543253..31b7c4a 100644 --- a/src/server/format-query-result.ts +++ b/src/server/format-query-result.ts @@ -46,8 +46,10 @@ export function formatSearchResultAsRow( const docNum = metadata['document_number']; const filename = metadata['filename']; const paper_number = - (typeof docNum === 'string' ? docNum : null) ?? - (typeof filename === 'string' ? filename.replace(/\.md$/i, '').toUpperCase() : null) ?? + (typeof docNum === 'string' && docNum.length > 0 ? docNum : null) ?? + (typeof filename === 'string' && filename.length > 0 + ? filename.replace(/\.md$/i, '').toUpperCase() + : null) ?? null; return { diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts index 9fc42f8..b4108eb 100644 --- a/src/server/tools/count-tool.ts +++ b/src/server/tools/count-tool.ts @@ -9,7 +9,7 @@ import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; const COUNT_RESPONSE_STATUS = 'success' as const; type CountResponse = | { - status: typeof COUNT_RESPONSE_STATUS; + status: 'success'; count: number; truncated: boolean; namespace: string; @@ -78,7 +78,7 @@ export function registerCountTool(server: McpServer): void { count, truncated, namespace, - metadata_filter: metadata_filter, + metadata_filter, }; return jsonResponse(response); } catch (error) { diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts index 785784c..ea3adc2 100644 --- a/src/server/tools/generate-urls-tool.ts +++ b/src/server/tools/generate-urls-tool.ts @@ -29,6 +29,7 @@ export function registerGenerateUrlsTool(server: McpServer): void { ), records: z .array(z.record(z.string(), z.unknown())) + .max(500) .describe( 'Array of records from retrieval results. Each item may be either metadata itself or an object containing a metadata field.' ), diff --git a/src/server/tools/query-documents-tool.ts b/src/server/tools/query-documents-tool.ts index 01d8a03..798d91d 100644 --- a/src/server/tools/query-documents-tool.ts +++ b/src/server/tools/query-documents-tool.ts @@ -111,7 +111,7 @@ export function registerQueryDocumentsTool(server: McpServer): void { status: 'success', query: query_text.trim(), namespace, - metadata_filter: metadata_filter ?? undefined, + metadata_filter, result_count: topDocuments.length, documents: topDocuments.map((doc) => ({ document_id: doc.document_id, diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts index 3294798..3797285 100644 --- a/src/server/tools/query-tool.ts +++ b/src/server/tools/query-tool.ts @@ -25,7 +25,7 @@ type QueryExecParams = { async function executeQuery(params: QueryExecParams) { const { query_text, namespace, top_k, use_reranking, metadata_filter, fields, mode } = params; try { - if (!query_text || !query_text.trim()) { + if (!query_text.trim()) { const response: QueryResponse = { status: 'error', message: 'Query text cannot be empty', @@ -128,8 +128,8 @@ export function registerQueryTool(server: McpServer): void { async (params) => executeQuery({ ...params, - top_k: params.top_k ?? 10, - use_reranking: params.use_reranking ?? true, + top_k: params.top_k, + use_reranking: params.use_reranking, mode: 'query', }) ); @@ -147,7 +147,7 @@ export function registerQueryTool(server: McpServer): void { async (params) => executeQuery({ ...params, - top_k: params.top_k ?? 10, + top_k: params.top_k, use_reranking: false, fields: params.fields?.length ? params.fields : [...FAST_QUERY_FIELDS], mode: 'query_fast', From b12c01440429fc7015fe1c4051d2e21f02c37c54 Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 13:28:22 +0800 Subject: [PATCH 18/20] #15-fix linter error --- src/index.ts | 5 +---- src/pinecone-client.ts | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index bf28d98..0b9163c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -102,10 +102,7 @@ async function main(): Promise { const options = parseArgs(); // Set log level (env + logger singleton so tools get correct level) - const rawLevel = - options.logLevel || - process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || - 'INFO'; + const rawLevel = options.logLevel || process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || 'INFO'; const logLevel = ( ['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) ? rawLevel : 'INFO' ) as LogLevel; diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 64a4295..07f4513 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -168,7 +168,7 @@ export class PineconeClient { if (!(key in metadataFields)) { metadataFields[key] = inferredType; } else if ( - metadataFields[key] === 'object' || metadataFields[key] === 'array' && + (metadataFields[key] === 'object' || metadataFields[key] === 'array') && inferredType === 'string[]' ) { // Prefer array type over generic object when we see it in another sample From eb593e9aa224edfea2b9f882b2dbc9b23026b74b Mon Sep 17 00:00:00 2001 From: zho Date: Sat, 21 Feb 2026 13:49:06 +0800 Subject: [PATCH 19/20] #15-saved fixed code --- src/server/metadata-filter.ts | 8 +++++--- src/server/tools/suggest-query-params-tool.ts | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index 98fb3a2..0e67c5f 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -50,10 +50,12 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n if (Array.isArray(value)) { for (let i = 0; i < value.length; i++) { - const parsed = metadataFilterSchema.safeParse(value[i]); - if (!parsed.success) { - return `Operator "${path[path.length - 1]}" at "${path.join('.')}" must use an array of filter objects.`; + const item = value[i]; + if (typeof item !== 'object' || item === null || Array.isArray(item)) { + return `Operator "${path[path.length - 1]}" at "${[...path, String(i)].join('.')}" must use an array of filter objects.`; } + const nestedError = validateMetadataFilter(item as Record); + if (nestedError) return nestedError; } return null; } diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts index f9a5f7b..b7b0ec7 100644 --- a/src/server/tools/suggest-query-params-tool.ts +++ b/src/server/tools/suggest-query-params-tool.ts @@ -47,9 +47,9 @@ export function registerSuggestQueryParamsTool(server: McpServer): void { }); } const response = { + ...result, status: 'success' as const, cache_hit, - ...result, }; return jsonResponse(response); } catch (error) { From e81bbc55de61d5ba22d043bc76993dae7f36b273 Mon Sep 17 00:00:00 2001 From: zho Date: Tue, 24 Feb 2026 10:21:21 +0800 Subject: [PATCH 20/20] #15-fix for review --- src/server/tools/count-tool.ts | 8 ++++---- src/server/url-generation.test.ts | 33 +++++++++++++++++++++++++++++++ src/server/url-generation.ts | 28 ++++++++++++++++++-------- 3 files changed, 57 insertions(+), 12 deletions(-) diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts index b4108eb..34ead93 100644 --- a/src/server/tools/count-tool.ts +++ b/src/server/tools/count-tool.ts @@ -23,10 +23,10 @@ export function registerCountTool(server: McpServer): void { 'count', { description: - 'Get the count of unique documents matching a metadata filter and semantic query. ' + - 'Use when the user asks for a count (e.g. "how many papers by Lakos?", "Lakos\'s paper count"). ' + + 'Get the number of unique documents matching a metadata filter and semantic query. ' + + 'Use when the user asks for a count (e.g. "how many papers by J. Lakos?", "John\'s paper count"). ' + 'Uses semantic (dense) search only and requests only document identifiers (no content) for performance. ' + - 'Returns unique document count (deduped by document_number/url/doc_id) up to 10,000; truncated=true if at least that many. ' + + 'Returns the number of unique documents (deduped by document_number/url/doc_id) up to 10,000; truncated=true if at least that many. ' + 'Mandatory flow: call suggest_query_params first, then count. ' + 'Use list_namespaces to discover namespace and metadata fields. ' + 'For count-by-metadata only, use a broad query_text (e.g. "paper" or "document"). ' + @@ -43,7 +43,7 @@ export function registerCountTool(server: McpServer): void { metadata_filter: metadataFilterSchema .optional() .describe( - 'Optional metadata filter. Use exact field names from list_namespaces. E.g. {"author": {"$in": ["John Lakos"]}} for author count.' + 'Optional metadata filter. Use exact field names from list_namespaces. E.g. {"author": {"$in": ["John Lakos", "J. Lakos"]}} for author count.' ), }, }, diff --git a/src/server/url-generation.test.ts b/src/server/url-generation.test.ts index ffe9a03..e1d7616 100644 --- a/src/server/url-generation.test.ts +++ b/src/server/url-generation.test.ts @@ -31,6 +31,39 @@ describe('generateUrlForNamespace', () => { expect(r.method).toBe('generated.mailing'); }); + it('generates mailing URL as list_name/message/doc_id when list_name present and doc_id does not contain it', () => { + const r = generateUrlForNamespace('mailing', { + list_name: 'boost-users', + doc_id: '12345', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-users/message/12345/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('uses msg_id when list_name present and doc_id missing', () => { + const r = generateUrlForNamespace('mailing', { + list_name: 'boost-announce', + msg_id: '67890', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-announce/message/67890/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('uses single-path form when doc_id contains list_name (no list_name/message split)', () => { + const r = generateUrlForNamespace('mailing', { + list_name: 'boost-users', + doc_id: 'boost-users@lists.boost.org/message/12345', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-users@lists.boost.org/message/12345/' + ); + expect(r.method).toBe('generated.mailing'); + }); + it('uses slack source when available', () => { const r = generateUrlForNamespace('slack-Cpplang', { source: 'https://app.slack.com/client/T123/C123/p123', diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index e47953a..32dd82a 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -19,14 +19,31 @@ function asString(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; } -/** Build a mailing-list URL from doc_id or thread_id (e.g. Boost archives). */ +/** + * Build a mailing-list URL (e.g. Boost archives). + * Two cases: + * 1. If metadata has list_name and doc_id (or msg_id) and the message id does not contain list_name, + * URL is https://lists.boost.org/archives/list/{list_name}/message/{doc_id}/ + * 2. Otherwise use doc_id or thread_id as the list path: .../list/{doc_id_or_thread_id}/ + */ function generatorMailing(metadata: Record): UrlGenerationResult { - const docIdOrThread = asString(metadata['doc_id']) ?? asString(metadata['thread_id']); + const listName = asString(metadata['list_name']); + const docId = asString(metadata['doc_id']) ?? asString(metadata['msg_id']); + const threadId = asString(metadata['thread_id']); + + if (listName && docId && !docId.includes(listName)) { + return { + url: `https://lists.boost.org/archives/list/${listName}/message/${docId}/`, + method: 'generated.mailing', + }; + } + + const docIdOrThread = docId ?? threadId; if (!docIdOrThread) { return { url: null, method: 'unavailable', - reason: 'mailing requires doc_id or thread_id to generate URL', + reason: 'mailing requires doc_id, msg_id, or thread_id to generate URL', }; } return { @@ -85,8 +102,3 @@ export function generateUrlForNamespace( reason: `URL generation is not supported for namespace "${namespace}"`, }; } - -/** Register a URL generator for a namespace (e.g. for custom indexes). */ -export function registerUrlGenerator(namespace: string, fn: UrlGenerator): void { - urlGenerators.set(namespace, fn); -}