diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5ec97af --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +node_modules +dist +.git +.github +.cursor +npm-debug.log +*.log +.env +.env.* +coverage +*.tsbuildinfo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d509fee..0d97076 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,7 @@ on: branches: [main] push: branches: [main] - workflow_call: # Allow this workflow to be called by other workflows + workflow_call: # Allow this workflow to be called by other workflows jobs: build-and-test: @@ -38,6 +38,9 @@ jobs: - name: Build run: npm run build + - name: Smoke test CLI + run: node dist/index.js --help + - name: Run tests run: npm test @@ -58,3 +61,6 @@ jobs: - name: Security audit run: npm audit --audit-level=moderate continue-on-error: true + + - name: Package dry run + run: npm pack --dry-run diff --git a/CHANGELOG.md b/CHANGELOG.md index b392f67..37f4a3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,26 +8,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- N/A -### Changed -- N/A +- `list_namespaces` tool: discovers all namespaces with record counts and sampled metadata fields +- `namespace_router` tool: ranks namespaces by relevance to a natural-language query +- `suggest_query_params` tool: mandatory flow gate that recommends fields and tool variant before queries +- `count` tool: counts unique documents matching a query, with `truncated` flag when results exceed `COUNT_TOP_K` +- `query_fast` tool: lightweight chunk-level search with minimal field set +- `query_detailed` tool: chunk-level search with optional semantic reranking +- `query` tool: unified query entry-point supporting `query_fast` and `query_detailed` modes +- `guided_query` tool: single-call orchestrator that runs routing → suggestion → execution, returning a `decision_trace` +- `generate_urls` tool: synthesizes URLs for records whose metadata lacks a `url` field +- `query_documents` tool: reassembles full documents from chunks grouped by document identifier +- Centralized structured logger (`src/logger.ts`) with level-gated stderr output and stack-trace capture +- Dockerfile for containerised deployment +- `About.md` project documentation covering architecture, tools, and operating principles -### Deprecated -- N/A +### Changed -### Removed -- N/A +- Modularised server into focused files under `src/server/` (tools, caches, formatters, suggestion flow) +- CI workflow updated: multi-node matrix (`18.x`, `20.x`, `22.x`), separate quality job with `continue-on-error` +- Replaced all `console.error` calls in `pinecone-client.ts` with typed `logInfo` / `logDebug` / `logError` ### Fixed -- N/A + +- Timestamp-based metadata filter now correctly handles numeric epoch values ### Security + - N/A ## [0.1.1] - 2026-01-27 ### Changed + - Enhanced TypeScript strict mode with additional compiler checks: - Added `noUncheckedIndexedAccess` for safer array/object access - Added `noImplicitOverride` to require explicit override keywords @@ -36,12 +49,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Simplified build script to use standard `tsc` command ### Fixed + - Fixed build script that was suppressing TypeScript compilation errors with `|| exit 0` - Fixed all type safety issues to comply with stricter TypeScript checks ## [0.1.0] - 2026-01-26 ### Added + - Initial release of TypeScript version - Feature parity with Python version - Production-ready implementation with: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1fcaac5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM node:20-bookworm-slim AS build + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci + +COPY . . +RUN npm run build + +FROM node:20-bookworm-slim AS runtime + +WORKDIR /app +ENV NODE_ENV=production + +# Create non-root runtime user +RUN useradd --create-home --uid 10001 mcpuser + +COPY package*.json ./ +RUN npm ci --omit=dev && npm cache clean --force + +COPY --from=build /app/dist ./dist +COPY README.md LICENSE CHANGELOG.md ./ + +USER mcpuser + +ENTRYPOINT ["node", "dist/index.js"] diff --git a/README.md b/README.md index c7e4786..fc814de 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,12 @@ The server requires a Pinecone API key and supports the following configuration ### Environment Variables -| Variable | Required | Default | Description | -|----------|----------|---------|-------------| -| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | -| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name | -| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | -| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | +| Variable | Required | Default | Description | +| ---------------------------------- | -------- | -------------------- | --------------------- | +| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | +| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name | +| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | +| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | ### Claude Desktop Configuration @@ -93,9 +93,12 @@ Or with explicit options: "args": [ "-y", "@will-cppa/pinecone-read-only-mcp", - "--api-key", "your-api-key-here", - "--index-name", "your-index-name", - "--rerank-model", "bge-reranker-v2-m3" + "--api-key", + "your-api-key-here", + "--index-name", + "your-index-name", + "--rerank-model", + "bge-reranker-v2-m3" ] } } @@ -147,6 +150,48 @@ node node_modules/@will-cppa/pinecone-read-only-mcp/dist/index.js --api-key YOUR --help, -h Show help message ``` +## Deployment + +### Production Readiness Defaults + +- Build now **fails fast** on TypeScript errors (`npm run build` no longer suppresses failures). +- CI validates typecheck, lint, format, build, smoke run, tests, and package dry-run. +- `list_namespaces` data is cached in-memory for 30 minutes to reduce repeated Pinecone calls. +- Query/count flow has guardrails (`suggest_query_params` before execution) to prevent wasteful calls. + +### Deploy with npm Package + +```bash +# install +npm i @will-cppa/pinecone-read-only-mcp + +# run +npx @will-cppa/pinecone-read-only-mcp --api-key YOUR_API_KEY +``` + +### Deploy with Docker + +```bash +# build image +docker build -t pinecone-read-only-mcp:latest . + +# run (stdio MCP server) +docker run --rm -i \ + -e PINECONE_API_KEY=YOUR_API_KEY \ + -e PINECONE_INDEX_NAME=rag-hybrid \ + pinecone-read-only-mcp:latest +``` + +### Release Gate (recommended) + +Before tagging/releasing: + +```bash +npm run release:check +``` + +This runs full CI-equivalent checks and validates publish contents with `npm pack --dry-run`. + ## API Documentation The server exposes the following tools via MCP: @@ -159,7 +204,11 @@ Discovers and lists all available namespaces in the configured Pinecone index, i **Returns:** JSON object with namespace details including available metadata fields +`metadata_fields` values represent inferred field types from sampled records. Common values include: +`string`, `number`, `boolean`, `string[]`, and `array`. + **Example response:** + ```json { "status": "success", @@ -186,29 +235,134 @@ Discovers and lists all available namespaces in the configured Pinecone index, i } ``` +### `suggest_query_params` + +Suggests which **fields** to request and which tool to use (`count`, `query_fast`, or `query_detailed`), based on the namespace’s schema (from `list_namespaces`) and the user’s natural language query. This is a mandatory flow step before `count`/`query` tools. + +**Parameters:** + +| Parameter | Type | Required | Description | +| ------------ | ------ | -------- | ----------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace to query (must match a name from `list_namespaces`) | +| `user_query` | string | Yes | User’s question or intent (e.g. "list papers by Lakos with titles", "how many papers by Wong?") | + +**Returns:** `suggested_fields` (only fields that exist in that namespace), `use_count_tool`, `recommended_tool`, `explanation`, and `namespace_found`. + +**Example response:** + +```json +{ + "status": "success", + "suggested_fields": ["document_number", "title", "url", "author"], + "use_count_tool": false, + "recommended_tool": "query_fast", + "explanation": "User asked for a list or browse; use minimal fields (no chunk_text) for smaller payload and cost.", + "namespace_found": true +} +``` + +Use `suggested_fields` as the `fields` parameter when calling query tools. + +### `guided_query` + +Single orchestrator tool that runs the full flow in one call: + +1. namespace routing (if namespace is omitted), +2. query param suggestion, +3. execution via `count`, `query_fast`, or `query_detailed`. + +It returns both the final result and a `decision_trace` for transparency. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +| ----------------- | ------- | -------- | ------- | ----------------------------------------------------------------------------------- | +| `user_query` | string | Yes | - | User question/intent | +| `namespace` | string | No | - | Optional explicit namespace | +| `metadata_filter` | object | No | - | Optional metadata filter | +| `top_k` | integer | No | `10` | Query result size for query paths (1-100) | +| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `query_fast`, `query_detailed` | +| `enrich_urls` | boolean | No | `true` | Auto-generate URLs for `mailing` and `slack-Cpplang` when `metadata.url` is missing | + +**Returns:** JSON containing `decision_trace` and `result`. + +### `generate_urls` + +Generates URLs for retrieved records when metadata does not contain `url` and URL is required. + +Supported namespaces: + +- `mailing` +- `slack-Cpplang` + +Rules: + +- **`mailing`**: uses `doc_id` or `thread_id` + Format: `https://lists.boost.org/archives/list/{doc_id_or_thread_id}/` +- **`slack-Cpplang`**: prefer `source` directly if present; otherwise use `team_id`, `channel_id`, and `doc_id` + `message_id = doc_id.replace('.', '')` + Format: `https://app.slack.com/{team_id}/{channel_id}/p{message_id}` + +**Parameters:** + +| Parameter | Type | Required | Description | +| ----------- | ------ | -------- | --------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace for URL-generation logic | +| `records` | array | Yes | Retrieved records; each item can be either metadata itself or an object with `metadata` field | + +**Returns:** Per-record generated URL, generation method, and reason if unavailable. + +### `count` + +Returns the **unique document count** matching a metadata filter and semantic query. Use for questions like "how many papers by Lakos?" instead of the `query` tool. For performance, the count tool uses **semantic (dense) search only** (no hybrid or lexical) and requests only document identifiers (`document_number`, `url`, `doc_id`)—no chunk content—then deduplicates by document. + +**Parameters:** + +| Parameter | Type | Required | Description | +| ----------------- | ------ | -------- | -------------------------------------------------------------------------------------------- | +| `namespace` | string | Yes | Namespace to count in (use `list_namespaces` to discover) | +| `query_text` | string | Yes | Search query; use a broad term (e.g. `"paper"`, `"document"`) when counting by metadata only | +| `metadata_filter` | object | No | Same operators as `query` (e.g. `{"author": {"$in": ["John Lakos"]}}` for wg21-papers) | + +**Returns:** JSON with `count` (unique documents, up to 10,000), and `truncated: true` if there are at least 10,000 matches. + +**Example response:** + +```json +{ + "status": "success", + "count": 42, + "truncated": false, + "namespace": "wg21-papers", + "metadata_filter": { "author": { "$in": ["John Lakos"] } } +} +``` + ### `query` -Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. +Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. For **count** questions, use the `count` tool instead. **Parameters:** -| Parameter | Type | Required | Default | Description | -|-----------|------|----------|---------|-------------| -| `query_text` | string | Yes | - | Search query text | -| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | -| `top_k` | integer | No | `10` | Number of results (1-100) | -| `use_reranking` | boolean | No | `true` | Enable semantic reranking | -| `metadata_filter` | object | No | - | Metadata filter to narrow results (e.g., `{"author": "John", "year": 2023}`) | +| Parameter | Type | Required | Default | Description | +| ----------------- | -------- | -------- | ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| `query_text` | string | Yes | - | Search query text | +| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | +| `top_k` | integer | No | `10` | Number of results (1-100) | +| `use_reranking` | boolean | No | `true` | Enable semantic reranking | +| `metadata_filter` | object | No | - | Metadata filter to narrow results (e.g., `{"author": "John", "year": 2023}`) | +| `fields` | string[] | No | - | Field names to return (e.g. `["document_number", "title", "url"]`). Omit for all fields; include `chunk_text` for content. Reduces payload and cost. | -**Returns:** JSON object with search results including content, relevance scores, and metadata +**Returns:** JSON object with search results (only requested fields when `fields` is set), relevance scores, and metadata **Example response:** + ```json { "status": "success", "query": "your search query", "namespace": "namespace1", - "metadata_filter": {"author": "John Doe"}, + "metadata_filter": { "author": "John Doe" }, "result_count": 10, "results": [ { @@ -230,16 +384,16 @@ Metadata filters allow you to narrow down search results based on document prope **Supported Operators (10 total):** -| Operator | Syntax | Description | Example | -|----------|--------|-------------|---------| -| Equal | `$eq` or value directly | Exact match | `{"status": "published"}` or `{"status": {"$eq": "published"}}` | -| Not Equal | `$ne` | Not equal to | `{"status": {"$ne": "draft"}}` | -| Greater Than | `$gt` | Greater than | `{"year": {"$gt": 2022}}` | -| Greater Than or Equal | `$gte` | Greater than or equal | `{"timestamp": {"$gte": 1704067200}}` | -| Less Than | `$lt` | Less than | `{"score": {"$lt": 0.5}}` | -| Less Than or Equal | `$lte` | Less than or equal | `{"priority": {"$lte": 3}}` | -| In Array | `$in` | Value is in array field | `{"tags": {"$in": ["cpp", "contracts"]}}` (only for array-type fields) | -| Not In Array | `$nin` | Value not in array field | `{"tags": {"$nin": ["draft", "archived"]}}` (only for array-type fields) | +| Operator | Syntax | Description | Example | +| --------------------- | ----------------------- | ------------------------ | ------------------------------------------------------------------------ | +| Equal | `$eq` or value directly | Exact match | `{"status": "published"}` or `{"status": {"$eq": "published"}}` | +| Not Equal | `$ne` | Not equal to | `{"status": {"$ne": "draft"}}` | +| Greater Than | `$gt` | Greater than | `{"year": {"$gt": 2022}}` | +| Greater Than or Equal | `$gte` | Greater than or equal | `{"timestamp": {"$gte": 1704067200}}` | +| Less Than | `$lt` | Less than | `{"score": {"$lt": 0.5}}` | +| Less Than or Equal | `$lte` | Less than or equal | `{"priority": {"$lte": 3}}` | +| In Array | `$in` | Value is in array field | `{"tags": {"$in": ["cpp", "contracts"]}}` (only for array-type fields) | +| Not In Array | `$nin` | Value not in array field | `{"tags": {"$nin": ["draft", "archived"]}}` (only for array-type fields) | **Filter Examples:** @@ -275,11 +429,14 @@ Metadata filters allow you to narrow down search results based on document prope ``` **Important Limitations:** + - **String fields require EXACT match** - No wildcards, partial matches, or substring searches - **Comma-separated strings**: If a field contains `"John Lakos, Herb Sutter"`, you cannot filter for just `"John Lakos"` - You must match the entire string: `{"author": "John Lakos, Herb Sutter"}` - To filter by individual authors, the data must be stored as an array field - **`$in` and `$nin` operators**: Only work on array-type fields, not comma-separated strings +- **Unsupported operators are rejected**: Unknown operators (for example `$regex`) return a validation error +- **`$in` and `$nin` must use arrays**: `{"tags": {"$in": "cpp"}}` is invalid; use `{"tags": {"$in": ["cpp"]}}` - Multiple conditions at the top level are combined with **AND** logic - Use comparison operators (`$gt`, `$gte`, `$lt`, `$lte`) for numeric and timestamp fields - Direct value assignment implies `$eq` (exact match) @@ -354,12 +511,14 @@ npm run dev -- --api-key YOUR_API_KEY ## Dependencies ### Production Dependencies + - [@modelcontextprotocol/sdk](https://www.npmjs.com/package/@modelcontextprotocol/sdk) - MCP SDK for TypeScript - [@pinecone-database/pinecone](https://www.npmjs.com/package/@pinecone-database/pinecone) - Pinecone client SDK - [zod](https://www.npmjs.com/package/zod) - TypeScript-first schema validation - [dotenv](https://www.npmjs.com/package/dotenv) - Environment variable management ### Development Dependencies + - [TypeScript](https://www.typescriptlang.org/) - Type-safe JavaScript - [ESLint](https://eslint.org/) - Code linting - [Prettier](https://prettier.io/) - Code formatting @@ -380,12 +539,14 @@ This TypeScript implementation provides the same functionality as the [Python ve ### API Key Issues If you see "Pinecone API key is required" error: + 1. Ensure `PINECONE_API_KEY` environment variable is set, OR 2. Pass `--api-key` option when running the server ### Index Not Found If you see index-related errors: + 1. Verify your index name is correct 2. Ensure your API key has access to the index 3. Check that both `your-index-name` and `your-index-name-sparse` indexes exist @@ -393,6 +554,7 @@ If you see index-related errors: ### Connection Issues If you experience connection issues: + 1. Check your internet connection 2. Verify Pinecone service status 3. Ensure firewall/proxy settings allow connections to Pinecone @@ -408,6 +570,7 @@ This project is licensed under the Boost Software License 1.0 - see the [LICENSE ## Acknowledgements This project uses: + - [Pinecone](https://www.pinecone.io/) for vector storage and retrieval - [Model Context Protocol](https://modelcontextprotocol.io/) for standardized AI integration - Hybrid search approach combining dense embeddings with sparse BM25-style retrieval @@ -420,6 +583,7 @@ This project uses: ## Support For issues and questions: + - GitHub Issues: [https://github.com/CppDigest/pinecone-read-only-mcp-typescript/issues](https://github.com/CppDigest/pinecone-read-only-mcp-typescript/issues) - Email: will@cppalliance.org diff --git a/package-lock.json b/package-lock.json index fab6064..e3254e8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -608,9 +608,9 @@ "license": "MIT" }, "node_modules/@eslint/js": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.2.tgz", - "integrity": "sha512-q1mjIoW1VX4IvSocvM/vbTiveKC4k9eLrajNEuSsmjymSDEbpGddtpfOoN7YGAqBK3NG+uqo8ia4PDTt8buCYA==", + "version": "9.39.3", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.39.3.tgz", + "integrity": "sha512-1B1VkCq6FuUNlQvlBYb+1jDu/gV297TIs/OeiaSR9l1H27SVW55ONE1e1Vp16NqP683+xEGzxYtv4XCiDPaQiw==", "dev": true, "license": "MIT", "engines": { @@ -1159,6 +1159,7 @@ "integrity": "sha512-m0jEgYlYz+mDJZ2+F4v8D1AyQb+QzsNqRuI7xg1VQX/KlKS0qT9r1Mo16yo5F/MtifXFgaofIFsdFMox2SxIbQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -1208,6 +1209,7 @@ "integrity": "sha512-IgSWvLobTDOjnaxAfDTIHaECbkNlAlKv2j5SjpB2v7QHKv1FIfjwMy8FsDbVfDX/KjmCmYICcw7uGaXLhtsLNg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.56.0", "@typescript-eslint/types": "8.56.0", @@ -1562,6 +1564,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1580,9 +1583,9 @@ } }, "node_modules/ajv": { - "version": "8.17.1", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", - "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", + "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -2033,11 +2036,12 @@ } }, "node_modules/eslint": { - "version": "9.39.2", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.2.tgz", - "integrity": "sha512-LEyamqS7W5HB3ujJyvi0HQK/dtVINZvd5mAAp9eT5S/ujByGjiZLCzPcHVzuXbpJDJF/cxwHlfceVUDZ2lnSTw==", + "version": "9.39.3", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.39.3.tgz", + "integrity": "sha512-VmQ+sifHUbI/IcSopBCF/HO3YiHQx/AVd3UVyYL6weuwW+HvON9VYn5l6Zl1WZzPWXPNZrSQpxwkkZ/VuvJZzg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -2045,7 +2049,7 @@ "@eslint/config-helpers": "^0.4.2", "@eslint/core": "^0.17.0", "@eslint/eslintrc": "^3.3.1", - "@eslint/js": "9.39.2", + "@eslint/js": "9.39.3", "@eslint/plugin-kit": "^0.4.1", "@humanfs/node": "^0.16.6", "@humanwhocodes/module-importer": "^1.0.1", @@ -2281,6 +2285,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -2628,10 +2633,11 @@ } }, "node_modules/hono": { - "version": "4.11.6", - "resolved": "https://registry.npmjs.org/hono/-/hono-4.11.6.tgz", - "integrity": "sha512-ofIiiHyl34SV6AuhE3YT2mhO5HRWokce+eUYE82TsP6z0/H3JeJcjVWEMSIAiw2QkjDOEpES/lYsg8eEbsLtdw==", + "version": "4.12.0", + "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.0.tgz", + "integrity": "sha512-NekXntS5M94pUfiVZ8oXXK/kkri+5WpX2/Ik+LVsl+uvw+soj4roXIsPqO+XsWrAw20mOzaXOZf3Q7PfB9A/IA==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } @@ -3156,6 +3162,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -3251,9 +3258,9 @@ } }, "node_modules/qs": { - "version": "6.14.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", - "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", + "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", "license": "BSD-3-Clause", "dependencies": { "side-channel": "^1.1.0" @@ -3680,6 +3687,7 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -3727,6 +3735,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -3800,6 +3809,7 @@ "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -4013,6 +4023,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index c411607..de5250e 100644 --- a/package.json +++ b/package.json @@ -36,10 +36,12 @@ "node": ">=18.0.0" }, "scripts": { - "build": "tsc", + "clean": "node -e \"const fs=require('fs'); fs.rmSync('dist',{recursive:true,force:true});\"", + "build": "npm run clean && npx tsc", "build:watch": "tsc --watch", "dev": "tsx watch src/index.ts", "start": "node dist/index.js", + "smoke": "npm run build && node dist/index.js --help", "test": "vitest run", "test:watch": "vitest", "test:search": "tsx scripts/test-search.ts", @@ -50,6 +52,7 @@ "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "typecheck": "tsc --noEmit", "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm test", + "release:check": "npm run ci && npm pack --dry-run --ignore-scripts", "ci:local": "bash scripts/ci-local.sh", "prepublishOnly": "npm run ci", "prepack": "npm run build" diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..47273f2 --- /dev/null +++ b/src/config.ts @@ -0,0 +1,5 @@ +/** + * Shared runtime config types. + */ + +export type LogLevel = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; diff --git a/src/constants.ts b/src/constants.ts index ef5faa8..0aa7ade 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -7,7 +7,29 @@ export const DEFAULT_RERANK_MODEL = 'bge-reranker-v2-m3'; export const DEFAULT_TOP_K = 10; export const MAX_TOP_K = 100; export const MIN_TOP_K = 1; -export const DEFAULT_NAMESPACE = 'mailing'; +/** Namespace and suggestion caches stay valid for 30 minutes. */ +export const FLOW_CACHE_TTL_MS = 30 * 60 * 1000; +/** + * Maximum hits fetched by the count tool to deduplicate into a document count. + * When the matching set exceeds this limit the count is capped; callers should + * check the `truncated: true` flag in the response to detect this condition. + */ +export const COUNT_TOP_K = 10_000; +/** + * Minimal fields fetched for count queries (no `chunk_text`) to reduce payload and cost. + * All three fields are tried as deduplication keys in priority order: + * 1. `document_number` — canonical document identifier used by most namespaces + * 2. `url` — used as a fallback document key when document_number is absent + * 3. `doc_id` — secondary fallback for namespaces that use a doc_id scheme + */ +export const COUNT_FIELDS = ['document_number', 'url', 'doc_id'] as const; +/** Default lightweight field set for fast queries. */ +export const FAST_QUERY_FIELDS = ['document_number', 'title', 'url', 'author', 'doc_id'] as const; +/** query_documents: default and max number of documents to return (reassembled from chunks). */ +export const DEFAULT_QUERY_DOCUMENTS_TOP_K = 5; +export const MAX_QUERY_DOCUMENTS_TOP_K = 20; +/** Max chunk hits to fetch when reassembling documents (then group by document). */ +export const QUERY_DOCUMENTS_MAX_CHUNKS = 500; export const SERVER_NAME = 'Pinecone Read-Only MCP'; export const SERVER_VERSION = '0.1.0'; @@ -19,7 +41,13 @@ Features: - Semantic Reranking: Uses BGE reranker model for improved precision - Dynamic Namespace Discovery: Automatically discovers available namespaces - Metadata Filtering: Supports optional metadata filters for refined searches +- Namespace Router: Suggests likely namespace(s) from natural-language intent +- Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count. +- URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url. +- Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization. Usage: -1. Use list_namespaces to discover available namespaces in the index -2. Use query to perform semantic search with hybrid retrieval and reranking`; +1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index +2. Optionally use namespace_router to choose candidate namespace(s) from user intent +3. Call suggest_query_params before query/count/query_documents tools (mandatory flow gate) to get suggested_fields and recommended tool +4. Use count for count questions, query_fast/query_detailed for chunk-level retrieval, or query_documents for full-document content`; diff --git a/src/index.ts b/src/index.ts index deecd46..0b9163c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,6 +12,8 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js' import { PineconeClient } from './pinecone-client.js'; import { setupServer, setPineconeClient } from './server.js'; import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL } from './constants.js'; +import type { LogLevel } from './config.js'; +import { setLogLevel } from './logger.js'; import * as dotenv from 'dotenv'; // Load environment variables @@ -24,6 +26,7 @@ interface CLIOptions { logLevel?: string; } +/** Parse argv into API key, index name, rerank model, and log level. */ function parseArgs(): CLIOptions { const args = process.argv.slice(2); const options: CLIOptions = {}; @@ -60,6 +63,7 @@ function parseArgs(): CLIOptions { return options; } +/** Print CLI usage and exit. */ function printHelp(): void { console.log(` Pinecone Read-Only MCP Server @@ -92,17 +96,17 @@ Examples: `); } +/** Initialize config, Pinecone client, MCP server, and stdio transport. */ async function main(): Promise { try { const options = parseArgs(); - // Set log level - const logLevel = - options.logLevel || - process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || - process.env['LOG_LEVEL'] || - 'INFO'; - process.env['LOG_LEVEL'] = logLevel; + // Set log level (env + logger singleton so tools get correct level) + const rawLevel = options.logLevel || process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || 'INFO'; + const logLevel = ( + ['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) ? rawLevel : 'INFO' + ) as LogLevel; + setLogLevel(logLevel); // Get API key const apiKey = options.apiKey || process.env['PINECONE_API_KEY']; diff --git a/src/logger.ts b/src/logger.ts new file mode 100644 index 0000000..8dc5392 --- /dev/null +++ b/src/logger.ts @@ -0,0 +1,82 @@ +/** + * Simple level-based logger for the MCP server. + * Logs to stderr; never logs secrets. Use for deploy/observability. + */ + +import type { LogLevel } from './config.js'; + +const LEVEL_ORDER: Record = { + DEBUG: 0, + INFO: 1, + WARN: 2, + ERROR: 3, +}; + +let currentLevel: LogLevel = 'INFO'; + +/** Set the minimum log level (DEBUG, INFO, WARN, ERROR). Messages below this level are dropped. */ +export function setLogLevel(level: LogLevel): void { + currentLevel = level; +} + +/** Return the current minimum log level. */ +export function getLogLevel(): LogLevel { + return currentLevel; +} + +/** True if the given level is at or above the current minimum. */ +function shouldLog(level: LogLevel): boolean { + return LEVEL_ORDER[level] >= LEVEL_ORDER[currentLevel]; +} + +/** Format a single log line: timestamp, level, message, and optional JSON data. */ +function formatMessage(level: LogLevel, msg: string, data?: unknown): string { + const ts = new Date().toISOString(); + const prefix = `[${ts}] [${level}]`; + if (data !== undefined) { + let serialized: string; + try { + serialized = JSON.stringify(data); + } catch { + serialized = String(data); + } + return `${prefix} ${msg} ${serialized}`; + } + return `${prefix} ${msg}`; +} + +/** Log a DEBUG-level message to stderr when the log level allows. */ +export function debug(msg: string, data?: unknown): void { + if (shouldLog('DEBUG')) { + console.error(formatMessage('DEBUG', msg, data)); + } +} + +/** Log an INFO-level message to stderr when the log level allows. */ +export function info(msg: string, data?: unknown): void { + if (shouldLog('INFO')) { + console.error(formatMessage('INFO', msg, data)); + } +} + +/** Log a WARN-level message to stderr when the log level allows. */ +export function warn(msg: string, data?: unknown): void { + if (shouldLog('WARN')) { + console.error(formatMessage('WARN', msg, data)); + } +} + +/** Log an ERROR-level message to stderr with optional error (message and stack). */ +export function error(msg: string, err?: unknown): void { + if (shouldLog('ERROR')) { + const detail = + err instanceof Error + ? { message: err.message, stack: err.stack } + : err !== undefined + ? String(err) + : undefined; + console.error( + formatMessage('ERROR', msg, detail !== undefined ? { error: detail } : undefined) + ); + } +} diff --git a/src/pinecone-client.test.ts b/src/pinecone-client.test.ts index 01cea7f..559e43b 100644 --- a/src/pinecone-client.test.ts +++ b/src/pinecone-client.test.ts @@ -1,5 +1,19 @@ import { describe, it, expect, beforeEach } from 'vitest'; import { PineconeClient } from './pinecone-client.js'; +import type { SearchableIndex, PineconeHit } from './types.js'; + +/** Test double: client with stubbable ensureIndexes and searchIndex for hybrid tests */ +type PineconeClientTestDouble = PineconeClient & { + ensureIndexes: () => Promise<{ denseIndex: SearchableIndex; sparseIndex: SearchableIndex }>; + searchIndex: ( + index: SearchableIndex, + query: string, + topK: number, + namespace?: string, + metadataFilter?: Record, + options?: { fields?: string[] } + ) => Promise; +}; describe('PineconeClient', () => { let client: PineconeClient; @@ -18,8 +32,8 @@ describe('PineconeClient', () => { }); it('should use environment variables as fallbacks', () => { - process.env.PINECONE_INDEX_NAME = 'env-index'; - process.env.PINECONE_RERANK_MODEL = 'env-model'; + process.env['PINECONE_INDEX_NAME'] = 'env-index'; + process.env['PINECONE_RERANK_MODEL'] = 'env-model'; const envClient = new PineconeClient({ apiKey: 'test-api-key', @@ -48,5 +62,121 @@ describe('PineconeClient', () => { }) ).rejects.toThrow('topK must be at least 1'); }); + + it('should continue hybrid search when one index fails', async () => { + const testClient = client as PineconeClientTestDouble; + + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); + + let searchCall = 0; + testClient.searchIndex = async () => { + searchCall += 1; + if (searchCall === 1) { + throw new Error('dense failure'); + } + return [ + { + _id: 'doc-1', + _score: 0.9, + fields: { chunk_text: 'hybrid content', author: 'tester' }, + }, + ]; + }; + + const results = await client.query({ + query: 'hybrid search', + namespace: 'test', + topK: 5, + useReranking: false, + }); + + expect(results).toHaveLength(1); + expect(results[0].content).toBe('hybrid content'); + expect(results[0].metadata.author).toBe('tester'); + }); + + it('should throw when both dense and sparse searches fail', async () => { + const testClient = client as PineconeClientTestDouble; + + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); + testClient.searchIndex = async () => { + throw new Error('index failure'); + }; + + await expect( + client.query({ + query: 'hybrid search', + namespace: 'test', + topK: 5, + useReranking: false, + }) + ).rejects.toThrow('Hybrid search failed: both dense and sparse index searches failed.'); + }); + }); + + describe('count', () => { + it('should return unique document count using semantic search only with minimal fields', async () => { + const testClient = client as PineconeClientTestDouble; + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); + + // Two chunks from doc A, one from doc B -> unique count 2 + testClient.searchIndex = async (_index, _query, _topK, _ns, _filter, options) => { + expect(options?.fields).toEqual(['document_number', 'url', 'doc_id']); + return [ + { + _id: 'c1', + _score: 1, + fields: { document_number: 'p1234r0', url: 'https://example.com/1' }, + }, + { + _id: 'c2', + _score: 0.9, + fields: { document_number: 'p1234r0', url: 'https://example.com/1' }, + }, + { + _id: 'c3', + _score: 0.8, + fields: { document_number: 'p5678r0', url: 'https://example.com/2' }, + }, + ]; + }; + + const result = await client.count({ + query: 'paper', + namespace: 'wg21-papers', + metadataFilter: { author: { $in: ['John Doe'] } }, + }); + + expect(result.count).toBe(2); + expect(result.truncated).toBe(false); + }); + + it('should set truncated when hit limit is reached', async () => { + const testClient = client as PineconeClientTestDouble; + testClient.ensureIndexes = async () => ({ + denseIndex: {} as SearchableIndex, + sparseIndex: {} as SearchableIndex, + }); + const manyHits: PineconeHit[] = Array.from({ length: 10000 }, (_, i) => ({ + _id: `id-${i}`, + _score: 1, + fields: { doc_id: `doc-${i}` }, + })); + testClient.searchIndex = async () => manyHits; + + const result = await client.count({ query: 'paper', namespace: 'ns' }); + + expect(result.count).toBe(10000); + expect(result.truncated).toBe(true); + }); }); }); diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 9252771..07f4513 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -7,8 +7,50 @@ */ import { Pinecone } from '@pinecone-database/pinecone'; -import type { PineconeClientConfig, SearchResult, PineconeHit, QueryParams } from './types.js'; -import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL, DEFAULT_TOP_K, MAX_TOP_K } from './constants.js'; +import { + debug as logDebug, + error as logError, + info as logInfo, + warn as logWarn, +} from './logger.js'; +import type { + PineconeClientConfig, + SearchResult, + PineconeHit, + QueryParams, + CountParams, + CountResult, + MergedHit, + NamespaceHandle, + SearchableIndex, + PineconeMetadataValue, +} from './types.js'; +import { + DEFAULT_INDEX_NAME, + DEFAULT_RERANK_MODEL, + DEFAULT_TOP_K, + MAX_TOP_K, + COUNT_TOP_K, + COUNT_FIELDS, +} from './constants.js'; + +/** + * Infers a human-readable metadata field type for namespace discovery. + * Distinguishes Pinecone-supported list type (string[]) from other arrays. + */ +function inferMetadataFieldType(value: unknown): string { + if (value === null || value === undefined) { + return 'unknown'; + } + if (Array.isArray(value)) { + if (value.length === 0) return 'array'; + if (value.every((item) => typeof item === 'string')) return 'string[]'; + return 'array'; + } + const t = typeof value; + if (t === 'string' || t === 'number' || t === 'boolean') return t; + return 'object'; +} export class PineconeClient { private apiKey: string; @@ -18,12 +60,11 @@ export class PineconeClient { // Lazy initialization private pc: Pinecone | null = null; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private denseIndex: any = null; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private sparseIndex: any = null; + private denseIndex: SearchableIndex | null = null; + private sparseIndex: SearchableIndex | null = null; private initialized = false; + /** Create a client with the given config; env vars override index name, rerank model, and top-k. */ constructor(config: PineconeClientConfig) { this.apiKey = config.apiKey; this.indexName = config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; @@ -44,7 +85,7 @@ export class PineconeClient { ); } this.pc = new Pinecone({ apiKey: this.apiKey }); - console.error('Pinecone client initialized'); + logInfo('Pinecone client initialized'); } return this.pc; } @@ -52,9 +93,11 @@ export class PineconeClient { /** * Ensure Pinecone indexes are initialized and return them */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private async ensureIndexes(): Promise<{ denseIndex: any; sparseIndex: any }> { - if (this.initialized) { + private async ensureIndexes(): Promise<{ + denseIndex: SearchableIndex; + sparseIndex: SearchableIndex; + }> { + if (this.initialized && this.denseIndex !== null && this.sparseIndex !== null) { return { denseIndex: this.denseIndex, sparseIndex: this.sparseIndex }; } @@ -62,12 +105,14 @@ export class PineconeClient { const denseName = this.indexName; const sparseName = `${this.indexName}-sparse`; - this.denseIndex = pc.index(denseName); - this.sparseIndex = pc.index(sparseName); + const dense = pc.index(denseName) as unknown as SearchableIndex; + const sparse = pc.index(sparseName) as unknown as SearchableIndex; + this.denseIndex = dense; + this.sparseIndex = sparse; this.initialized = true; - console.error(`Connected to indexes: ${denseName} and ${sparseName}`); - return { denseIndex: this.denseIndex, sparseIndex: this.sparseIndex }; + logInfo(`Connected to indexes: ${denseName} and ${sparseName}`); + return { denseIndex: dense, sparseIndex: sparse }; } /** @@ -87,43 +132,54 @@ export class PineconeClient { const { denseIndex } = await this.ensureIndexes(); // Get index stats to find namespaces - const stats = await denseIndex.describeIndexStats(); + const stats = denseIndex.describeIndexStats + ? await denseIndex.describeIndexStats() + : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; - console.error(`Found ${namespaces.length} namespace(s)`); + logInfo(`Found ${namespaces.length} namespace(s)`); // Get metadata info for each namespace by sampling records const namespacesInfo = await Promise.all( namespaces.map(async (ns: string) => { try { - const recordCount = stats.namespaces?.[ns]?.recordCount || 0; + const recordCount = stats?.namespaces?.[ns]?.recordCount || 0; const metadataFields: Record = {}; // Sample a few records to discover metadata fields - if (recordCount > 0) { + if (recordCount > 0 && denseIndex.namespace) { try { - const nsObj = denseIndex.namespace(ns); - // Query with a dummy vector to get some sample records - const sampleQuery = await nsObj.query({ - topK: 5, - vector: Array(stats.dimension || 1536).fill(0), - includeMetadata: true, - }); - - // Collect unique metadata fields and infer types + const nsObj: NamespaceHandle = denseIndex.namespace(ns); + const sampleQuery = + typeof nsObj.query === 'function' + ? await nsObj.query({ + topK: 5, + vector: Array(stats?.dimension ?? 1536).fill(0), + includeMetadata: true, + }) + : { matches: undefined }; + + // Collect unique metadata fields and infer types (including string[]) if (sampleQuery?.matches) { - sampleQuery.matches.forEach((match: any) => { + sampleQuery.matches.forEach((match: { metadata?: Record }) => { if (match.metadata) { Object.entries(match.metadata).forEach(([key, value]) => { + const inferredType = inferMetadataFieldType(value); if (!(key in metadataFields)) { - metadataFields[key] = typeof value; + metadataFields[key] = inferredType; + } else if ( + (metadataFields[key] === 'object' || metadataFields[key] === 'array') && + inferredType === 'string[]' + ) { + // Prefer array type over generic object when we see it in another sample + metadataFields[key] = inferredType; } }); } }); } } catch (queryError) { - console.error(`Error sampling records for namespace ${ns}:`, queryError); + logError(`Error sampling records for namespace ${ns}`, queryError); } } @@ -133,7 +189,7 @@ export class PineconeClient { metadata: metadataFields, }; } catch (error) { - console.error(`Error processing namespace ${ns}:`, error); + logError(`Error processing namespace ${ns}`, error); return { namespace: ns, recordCount: 0, @@ -145,46 +201,76 @@ export class PineconeClient { return namespacesInfo; } catch (error) { - console.error('Error listing namespaces:', error); + logError('Error listing namespaces', error); return []; } } /** - * Search a Pinecone index using text query with optional metadata filtering + * Search a Pinecone index using text query with optional metadata filtering. + * When options.fields is set, only those fields are requested (e.g. for count: no chunk_text). */ private async searchIndex( - // eslint-disable-next-line @typescript-eslint/no-explicit-any - index: any, + index: SearchableIndex, query: string, topK: number, namespace?: string, - metadataFilter?: Record + metadataFilter?: Record, + options?: { fields?: string[] } ): Promise { + // Build query payload in the same shape as Python implementation. + const queryPayload: Record = { + top_k: topK, + inputs: { text: query }, + }; + + // Include filter when explicitly provided (matches Python behavior). + if (metadataFilter !== undefined) { + queryPayload['filter'] = metadataFilter; + logDebug('Applying metadata filter', metadataFilter); + } + try { - // Get namespace object - const ns = namespace ? index.namespace(namespace) : index; + // Preferred path: Pinecone search API. + if (typeof index.search === 'function') { + const searchOpts: { + namespace?: string; + query: Record; + fields?: string[]; + } = { + namespace, + query: queryPayload, + }; + if (options?.fields?.length) { + searchOpts.fields = options.fields; + } + const result = await index.search(searchOpts); + return result?.result?.hits || []; + } - // Use searchRecords API (Pinecone v5+) - const queryParams: any = { + // Backward-compatible fallback for older API shapes. + const target = namespace && index.namespace ? index.namespace(namespace) : index; + const queryParams: { query: Record; fields?: string[] } = { query: { topK, inputs: { text: query }, }, }; - - // Add metadata filter if provided - if (metadataFilter && Object.keys(metadataFilter).length > 0) { - queryParams.query.filter = metadataFilter; - console.error('Applying metadata filter:', JSON.stringify(metadataFilter)); + if (metadataFilter !== undefined) { + queryParams.query['filter'] = metadataFilter; } - - const result = await ns.searchRecords(queryParams); - + if (options?.fields?.length) { + queryParams.fields = options.fields; + } + const result = target.searchRecords + ? await target.searchRecords(queryParams) + : { result: { hits: [] as PineconeHit[] } }; return result?.result?.hits || []; } catch (error) { - console.error('Error searching index:', error); - return []; + const errorMessage = error instanceof Error ? error.message : String(error); + throw new Error( + `Pinecone search failed for namespace "${namespace ?? 'default'}": ${errorMessage}` + ); } } @@ -193,26 +279,26 @@ export class PineconeClient { * * Uses the higher score when duplicates are found. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any - private mergeResults(denseHits: PineconeHit[], sparseHits: PineconeHit[]): any[] { - const deduped: Record = {}; + private mergeResults(denseHits: PineconeHit[], sparseHits: PineconeHit[]): MergedHit[] { + const deduped: Record = {}; for (const hit of [...denseHits, ...sparseHits]) { const hitId = hit._id || ''; const hitScore = hit._score || 0; - if (hitId in deduped && (deduped[hitId]._score || 0) >= hitScore) { + const existing = deduped[hitId]; + if (existing !== undefined && (existing._score || 0) >= hitScore) { continue; } - const hitMetadata: Record = {}; + const hitMetadata: Record = {}; let content = ''; for (const [key, value] of Object.entries(hit.fields || {})) { if (key === 'chunk_text') { - content = value; + content = typeof value === 'string' ? value : ''; } else { - hitMetadata[key] = value; + hitMetadata[key] = value as PineconeMetadataValue; } } @@ -232,8 +318,7 @@ export class PineconeClient { */ private async rerankResults( query: string, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - results: any[], + results: MergedHit[], topN: number ): Promise { if (!results || results.length === 0) { @@ -243,16 +328,27 @@ export class PineconeClient { const pc = this.ensureClient(); try { - const rerankResult = await pc.inference.rerank(this.rerankModel, query, results, { - topN, - rankFields: ['chunk_text'], - returnDocuments: true, - parameters: { truncate: 'END' }, - }); + const rerankResult = await pc.inference.rerank( + this.rerankModel, + query, + // The Pinecone SDK types constrain document values to `Record`, + // but the underlying HTTP API accepts any JSON value. We pass MergedHit objects + // (metadata may contain number/boolean/string[]) and only `chunk_text` — which is + // always a string — is accessed via rankFields. The double cast via `as unknown` + // is intentional: it bypasses the SDK's over-narrow type without stringifying + // metadata values that we need to read back from the returned documents. + results as unknown as (string | Record)[], + { + topN, + rankFields: ['chunk_text'], + returnDocuments: true, + parameters: { truncate: 'END' }, + } + ); const reranked: SearchResult[] = []; for (const item of rerankResult.data || []) { - const document = item.document || {}; + const document = (item.document || {}) as MergedHit; reranked.push({ id: document['_id'] || '', content: document['chunk_text'] || '', @@ -263,7 +359,7 @@ export class PineconeClient { } return reranked; } catch (error) { - console.error('Error reranking results:', error); + logError('Error reranking results', error); // Fall back to returning unreranked results return results.slice(0, topN).map((result) => ({ id: result._id || '', @@ -282,7 +378,14 @@ export class PineconeClient { * and optionally reranks using the configured reranking model. */ async query(params: QueryParams): Promise { - const { query, topK: requestedTopK, namespace, metadataFilter, useReranking = true } = params; + const { + query, + topK: requestedTopK, + namespace, + metadataFilter, + useReranking = true, + fields: requestedFields, + } = params; // Validate inputs if (!query || !query.trim()) { @@ -293,19 +396,41 @@ export class PineconeClient { if (topK < 1) { throw new Error('topK must be at least 1'); } + if (topK > MAX_TOP_K) { - topK = MAX_TOP_K; // Cap at 100 for performance + topK = MAX_TOP_K; } + // When reranking, Pinecone requires chunk_text in returned fields; add it if user specified fields without it + const searchFields = + requestedFields?.length && useReranking && !requestedFields.includes('chunk_text') + ? [...requestedFields, 'chunk_text'] + : requestedFields; + // Ensure indexes are ready const { denseIndex, sparseIndex } = await this.ensureIndexes(); + const searchOptions = searchFields?.length ? { fields: searchFields } : undefined; + // Perform hybrid search - const [denseHits, sparseHits] = await Promise.all([ - this.searchIndex(denseIndex, query, topK, namespace, metadataFilter), - this.searchIndex(sparseIndex, query, topK, namespace, metadataFilter), + const [denseResult, sparseResult] = await Promise.allSettled([ + this.searchIndex(denseIndex, query, topK, namespace, metadataFilter, searchOptions), + this.searchIndex(sparseIndex, query, topK, namespace, metadataFilter, searchOptions), ]); + const denseHits = denseResult.status === 'fulfilled' ? denseResult.value : []; + const sparseHits = sparseResult.status === 'fulfilled' ? sparseResult.value : []; + + if (denseResult.status === 'rejected') { + logError('Dense index search failed', denseResult.reason); + } + if (sparseResult.status === 'rejected') { + logError('Sparse index search failed', sparseResult.reason); + } + if (denseResult.status === 'rejected' && sparseResult.status === 'rejected') { + throw new Error('Hybrid search failed: both dense and sparse index searches failed.'); + } + // Merge results const mergedResults = this.mergeResults(denseHits, sparseHits); @@ -323,10 +448,63 @@ export class PineconeClient { })); } - console.error( + logInfo( `Retrieved ${documents.length} documents from hybrid search (dense: ${denseHits.length}, sparse: ${sparseHits.length})` ); return documents; } + + /** + * Return the number of unique documents matching the query and optional metadata filter. + * Uses semantic search only (dense index), requests minimal fields (document_number, url, doc_id) + * to avoid transferring chunk content, and deduplicates by document for a document-level count. + */ + async count(params: CountParams): Promise { + if (!params.query || !params.query.trim()) { + throw new Error('Query cannot be empty'); + } + const { denseIndex } = await this.ensureIndexes(); + + const hits = await this.searchIndex( + denseIndex, + params.query, + COUNT_TOP_K, + params.namespace, + params.metadataFilter, + { fields: [...COUNT_FIELDS] } + ); + + const docKeys = new Set(); + let idFallbackCount = 0; + for (const hit of hits) { + const fields = hit.fields || {}; + const docNumber = fields['document_number']; + const url = fields['url']; + const docId = fields['doc_id']; + const docKey = + (typeof docNumber === 'string' ? docNumber : undefined) ?? + (typeof url === 'string' ? url : undefined) ?? + (typeof docId === 'string' ? docId : undefined); + if (docKey !== undefined) { + docKeys.add(docKey); + } else { + // Fall back to chunk ID — this yields a chunk count, not a document count + idFallbackCount++; + docKeys.add(hit._id ?? ''); + } + } + if (idFallbackCount > 0) { + logWarn( + `count(): ${idFallbackCount} hit(s) in namespace "${params.namespace}" had none of the ` + + `identifier fields (${COUNT_FIELDS.join(', ')}); fell back to chunk ID — result may overcount documents` + ); + } + + const count = docKeys.size; + return { + count, + truncated: hits.length >= COUNT_TOP_K, + }; + } } diff --git a/src/server.test.ts b/src/server.test.ts new file mode 100644 index 0000000..b431bf6 --- /dev/null +++ b/src/server.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from 'vitest'; +import { validateMetadataFilter, suggestQueryParams } from './server.js'; + +describe('suggestQueryParams', () => { + const wg21Fields = { + author: 'string[]', + chunk_text: 'string', + document_number: 'string', + title: 'string', + type: 'string', + url: 'string', + }; + + it('suggests count tool and minimal fields for count-style queries', () => { + const r = suggestQueryParams(wg21Fields, 'How many papers by Lakos?'); + expect(r.use_count_tool).toBe(true); + expect(r.suggested_fields).toContain('document_number'); + expect(r.suggested_fields).toContain('url'); + expect(r.namespace_found).toBe(true); + }); + + it('suggests chunk_text for content-style queries', () => { + const r = suggestQueryParams(wg21Fields, 'What does the paper say about contracts?'); + expect(r.use_count_tool).toBe(false); + expect(r.suggested_fields).toContain('chunk_text'); + expect(r.namespace_found).toBe(true); + }); + + it('suggests minimal fields for list-style queries', () => { + const r = suggestQueryParams(wg21Fields, 'List papers by Michael Wong with titles and links'); + expect(r.use_count_tool).toBe(false); + expect(r.suggested_fields).not.toContain('chunk_text'); + expect(r.suggested_fields).toContain('title'); + expect(r.suggested_fields).toContain('url'); + expect(r.namespace_found).toBe(true); + }); + + it('returns namespace_found false when metadata is null', () => { + const r = suggestQueryParams(null, 'list papers'); + expect(r.namespace_found).toBe(false); + expect(r.suggested_fields).toEqual([]); + }); +}); + +describe('validateMetadataFilter', () => { + it('accepts direct scalar equality filters', () => { + const result = validateMetadataFilter({ + status: 'published', + year: 2024, + featured: true, + }); + + expect(result).toBeNull(); + }); + + it('accepts supported comparison operators and array operators', () => { + const result = validateMetadataFilter({ + year: { $gte: 2020, $lte: 2026 }, + tags: { $in: ['cpp', 'contracts'] }, + }); + + expect(result).toBeNull(); + }); + + it('rejects unsupported operators', () => { + const result = validateMetadataFilter({ + year: { $regex: '^202' }, + }); + + expect(result).toContain('Unsupported filter operator'); + }); + + it('rejects non-array values for $in/$nin', () => { + const result = validateMetadataFilter({ + tags: { $in: 'cpp' }, + }); + + expect(result).toContain('must use an array of primitive values'); + }); +}); diff --git a/src/server.ts b/src/server.ts index f88464b..a9beb82 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,53 +1,27 @@ /** * Pinecone Read-Only MCP Server * - * This module implements a Model Context Protocol (MCP) server that provides - * semantic search capabilities over Pinecone vector databases using hybrid - * search (dense + sparse) with reranking. + * Keeps server composition slim by delegating validation, heuristics, + * and tool registration into dedicated modules. */ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { z } from 'zod'; -import { PineconeClient } from './pinecone-client.js'; -import { - SERVER_NAME, - SERVER_VERSION, - SERVER_INSTRUCTIONS, - MIN_TOP_K, - MAX_TOP_K, -} from './constants.js'; -import type { QueryResponse } from './types.js'; - -// Recursive Zod schema for Pinecone metadata filters -// Supports nested objects with operators like {"timestamp": {"$gte": 123}} -// Using z.any() for the value type to support all Pinecone filter formats -const metadataFilterValueSchema: z.ZodType = z.lazy(() => - z.union([ - z.string(), - z.number(), - z.boolean(), - z.array(z.string()), - z.array(z.number()), - z.record(z.string(), metadataFilterValueSchema), // Recursive for nested operators - ]) -); - -const metadataFilterSchema = z.record(z.string(), metadataFilterValueSchema); - -// Global Pinecone client (initialized lazily) -let pineconeClient: PineconeClient | null = null; - -function getPineconeClient(): PineconeClient { - if (!pineconeClient) { - throw new Error('Pinecone client not initialized. Call setPineconeClient first.'); - } - return pineconeClient; -} - -export function setPineconeClient(client: PineconeClient): void { - pineconeClient = client; -} - +import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js'; +import { registerCountTool } from './server/tools/count-tool.js'; +import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js'; +import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js'; +import { registerListNamespacesTool } from './server/tools/list-namespaces-tool.js'; +import { registerNamespaceRouterTool } from './server/tools/namespace-router-tool.js'; +import { registerQueryDocumentsTool } from './server/tools/query-documents-tool.js'; +import { registerQueryTool } from './server/tools/query-tool.js'; +import { registerSuggestQueryParamsTool } from './server/tools/suggest-query-params-tool.js'; + +export { setPineconeClient } from './server/client-context.js'; +export { validateMetadataFilter } from './server/metadata-filter.js'; +export { suggestQueryParams } from './server/query-suggestion.js'; +export type { SuggestQueryParamsResult } from './server/query-suggestion.js'; + +/** Create and configure the MCP server with all tools; returns the server instance. */ export async function setupServer(): Promise { const server = new McpServer( { @@ -59,204 +33,14 @@ export async function setupServer(): Promise { } ); - // Tool: list_namespaces - server.registerTool( - 'list_namespaces', - { - description: - 'List all available namespaces in the Pinecone index with their metadata fields and record counts. ' + - 'Returns detailed information about each namespace including available metadata fields that can be used for filtering in queries. ' + - 'Use this tool first to discover which namespaces exist and what metadata fields are available for filtering.', - inputSchema: {}, - }, - async () => { - try { - const client = getPineconeClient(); - const namespacesInfo = await client.listNamespacesWithMetadata(); - - const response = { - status: 'success', - count: namespacesInfo.length, - namespaces: namespacesInfo.map((ns) => ({ - name: ns.namespace, - record_count: ns.recordCount, - metadata_fields: ns.metadata, - })), - }; - - return { - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('Error listing namespaces:', error); - - const response = { - status: 'error', - message: - process.env['LOG_LEVEL'] === 'DEBUG' ? errorMessage : 'Failed to list namespaces', - }; - - return { - isError: true, - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } - } - ); - - // Tool: query - server.registerTool( - 'query', - { - description: - 'Search the Pinecone vector database using hybrid semantic search with optional metadata filtering. ' + - 'Performs a hybrid search combining dense and sparse embeddings for better recall, with optional reranking for improved precision. ' + - 'Supports natural language queries and returns the most relevant documents based on semantic similarity. ' + - 'IMPORTANT: Use metadata_filter to narrow results. First call list_namespaces to discover available metadata fields. ' + - 'Metadata filters support 10 operators: $eq/==, $ne/!=, $gt/>, $gte/>=, $lt/<, $lte/<=, $in (array fields only), $nin (array fields only). ' + - 'NOTE: String fields require exact match - no wildcards. For comma-separated strings, you cannot filter by individual values. ' + - 'Use comparison operators for numeric/timestamp fields. Multiple top-level conditions use AND logic.', - inputSchema: { - query_text: z.string().describe('Search query text. Be specific for better results.'), - namespace: z - .string() - .describe( - 'Namespace to search within. Use list_namespaces tool to discover available namespaces in the index.' - ), - top_k: z - .number() - .int() - .min(MIN_TOP_K) - .max(MAX_TOP_K) - .default(10) - .describe('Number of results to return (1-100). Default: 10'), - use_reranking: z - .boolean() - .default(true) - .describe( - 'Whether to use semantic reranking for better relevance. Slower but more accurate. Default: true' - ), - metadata_filter: z - .record(z.string(), metadataFilterSchema) - .optional() - .describe( - 'Optional metadata filter to narrow down search results. Use exact field names from list_namespaces. ' + - 'Supports 10 operators: ' + - '$eq (==), $ne (!=), $gt (>), $gte (>=), $lt (<), $lte (<=), $in (array field contains value), $nin (array field not contains). ' + - 'IMPORTANT: String fields require EXACT match - no wildcards/partial matches. For comma-separated values (like authors), use exact full string. ' + - 'Examples: ' + - '{"author": "John Lakos"} - exact author match (single author only), ' + - '{"year": {"$gte": 2023}} - year >= 2023, ' + - '{"timestamp": {"$gt": 1704067200, "$lt": 1735689600}} - timestamp range, ' + - '{"status": "published"} - exact match (implicit $eq), ' + - '{"tags": {"$in": ["cpp", "contracts"]}} - tags array contains value (only works if tags is array field). ' + - 'Multiple conditions at top level are combined with AND logic.' - ), - }, - }, - async (params) => { - try { - const { query_text, namespace, top_k = 10, use_reranking = true, metadata_filter } = params; - - // Validate query - if (!query_text || !query_text.trim()) { - const response: QueryResponse = { - status: 'error', - message: 'Query text cannot be empty', - }; - - return { - isError: true, - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } - - // Log filter for debugging - if (metadata_filter) { - console.error('Received metadata filter:', JSON.stringify(metadata_filter, null, 2)); - } - - const client = getPineconeClient(); - const results = await client.query({ - query: query_text.trim(), - topK: top_k, - namespace, - useReranking: use_reranking, - metadataFilter: metadata_filter, - }); - - // Format results for output - const formattedResults = results.map((doc) => ({ - paper_number: - doc.metadata['document_number'] || - (doc.metadata['filename'] as string)?.replace('.md', '').toUpperCase() || - null, - title: doc.metadata['title'] || '', - author: doc.metadata['author'] || '', - url: doc.metadata['url'] || '', - content: doc.content.substring(0, 2000), // Truncate for readability - score: Math.round(doc.score * 10000) / 10000, - reranked: doc.reranked, - metadata: doc.metadata, // Include all metadata fields (including timestamp) - })); - - const response: QueryResponse = { - status: 'success', - query: query_text, - namespace, - metadata_filter: metadata_filter, - result_count: formattedResults.length, - results: formattedResults, - }; - - return { - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } catch (error) { - const errorMessage = error instanceof Error ? error.message : String(error); - console.error('Error executing query:', error); - - const response: QueryResponse = { - status: 'error', - message: - process.env['LOG_LEVEL'] === 'DEBUG' - ? errorMessage - : 'An error occurred while processing your query', - }; - - return { - isError: true, - content: [ - { - type: 'text' as const, - text: JSON.stringify(response, null, 2), - }, - ], - }; - } - } - ); + registerListNamespacesTool(server); + registerNamespaceRouterTool(server); + registerSuggestQueryParamsTool(server); + registerCountTool(server); + registerQueryTool(server); + registerQueryDocumentsTool(server); + registerGuidedQueryTool(server); + registerGenerateUrlsTool(server); return server; } diff --git a/src/server/client-context.ts b/src/server/client-context.ts new file mode 100644 index 0000000..f2f759f --- /dev/null +++ b/src/server/client-context.ts @@ -0,0 +1,17 @@ +import { PineconeClient } from '../pinecone-client.js'; + +// Global Pinecone client (initialized lazily) +let pineconeClient: PineconeClient | null = null; + +/** Return the shared Pinecone client; throws if setPineconeClient has not been called. */ +export function getPineconeClient(): PineconeClient { + if (!pineconeClient) { + throw new Error('Pinecone client not initialized. Call setPineconeClient first.'); + } + return pineconeClient; +} + +/** Set the shared Pinecone client used by all MCP tools. */ +export function setPineconeClient(client: PineconeClient): void { + pineconeClient = client; +} diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts new file mode 100644 index 0000000..31b7c4a --- /dev/null +++ b/src/server/format-query-result.ts @@ -0,0 +1,79 @@ +/** + * Shared formatting of Pinecone SearchResult into QueryResponse result rows. + * Used by query tool and guided_query to avoid duplicated paper_number/title/author/url logic. + */ + +import type { PineconeMetadataValue, SearchResult } from '../types.js'; +import { generateUrlForNamespace } from './url-generation.js'; + +const DEFAULT_CONTENT_MAX_LENGTH = 2000; + +export interface QueryResultRow { + paper_number: string | null; + title: string; + author: string; + url: string; + content: string; + score: number; + reranked: boolean; + metadata?: Record; +} + +/** + * Format a single search result into a QueryResponse result row. + * Optionally enrich url using the namespace URL generator when metadata.url is missing (if supported). + */ +export function formatSearchResultAsRow( + doc: SearchResult, + options?: { + namespace?: string; + enrichUrls?: boolean; + contentMaxLength?: number; + } +): QueryResultRow { + const contentMaxLength = options?.contentMaxLength ?? DEFAULT_CONTENT_MAX_LENGTH; + const metadata = { ...doc.metadata } as Record; + + if (options?.enrichUrls && options?.namespace) { + const generated = generateUrlForNamespace(options.namespace, metadata); + const existingUrl = metadata['url']; + const urlIsBlank = typeof existingUrl !== 'string' || existingUrl.trim() === ''; + if (generated.url && urlIsBlank) { + metadata['url'] = generated.url; + } + } + + const docNum = metadata['document_number']; + const filename = metadata['filename']; + const paper_number = + (typeof docNum === 'string' && docNum.length > 0 ? docNum : null) ?? + (typeof filename === 'string' && filename.length > 0 + ? filename.replace(/\.md$/i, '').toUpperCase() + : null) ?? + null; + + return { + paper_number, + title: String(metadata['title'] ?? ''), + author: String(metadata['author'] ?? ''), + url: String(metadata['url'] ?? ''), + content: doc.content.substring(0, contentMaxLength), + score: Math.round(doc.score * 10000) / 10000, + reranked: doc.reranked, + metadata, + }; +} + +/** + * Format an array of search results into QueryResponse result rows. + */ +export function formatQueryResultRows( + results: SearchResult[], + options?: { + namespace?: string; + enrichUrls?: boolean; + contentMaxLength?: number; + } +): QueryResultRow[] { + return results.map((doc) => formatSearchResultAsRow(doc, options)); +} diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts new file mode 100644 index 0000000..0e67c5f --- /dev/null +++ b/src/server/metadata-filter.ts @@ -0,0 +1,108 @@ +import { z } from 'zod'; + +// Recursive Zod schema for Pinecone metadata filters +// Supports nested objects with operators like {"timestamp": {"$gte": 123}} +const metadataFilterValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.array(z.string()), + z.array(z.number()), + z.array(z.lazy(() => metadataFilterSchema)), + z.record(z.string(), metadataFilterValueSchema), // Recursive for nested operators + ]) +); + +export const metadataFilterSchema = z.record(z.string(), metadataFilterValueSchema); +const ALLOWED_FILTER_OPERATORS = new Set([ + '$eq', + '$ne', + '$gt', + '$gte', + '$lt', + '$lte', + '$in', + '$nin', + '$and', + '$or', +]); + +/** True if value is a string, number, or boolean (allowed for $eq, $gt, etc.). */ +function isPrimitiveFilterValue(value: unknown): boolean { + return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; +} + +/** True if value is an array of string (required for $in/$nin). */ +function isPrimitiveArray(value: unknown): boolean { + return Array.isArray(value) && value.every((item) => typeof item === 'string'); +} + +/** Recursively validate a filter value; returns an error string or null if valid. */ +function validateMetadataFilterValue(value: unknown, path: string[]): string | null { + if (value === null || value === undefined) { + return `Invalid null/undefined at "${path.join('.')}".`; + } + + if (isPrimitiveFilterValue(value) || isPrimitiveArray(value)) { + return null; + } + + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + const item = value[i]; + if (typeof item !== 'object' || item === null || Array.isArray(item)) { + return `Operator "${path[path.length - 1]}" at "${[...path, String(i)].join('.')}" must use an array of filter objects.`; + } + const nestedError = validateMetadataFilter(item as Record); + if (nestedError) return nestedError; + } + return null; + } + + if (typeof value !== 'object') { + return `Unsupported filter value at "${path.join('.')}".`; + } + + for (const [key, nestedValue] of Object.entries(value)) { + if (!key.startsWith('$')) { + return `Unsupported filter operator "${key}" at "${path.join('.')}".`; + } + if (!ALLOWED_FILTER_OPERATORS.has(key)) { + return `Unsupported filter operator "${key}" at "${path.join('.')}".`; + } + if ((key === '$in' || key === '$nin') && !isPrimitiveArray(nestedValue)) { + return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; + } + if ( + (key === '$eq' || + key === '$ne' || + key === '$gt' || + key === '$gte' || + key === '$lt' || + key === '$lte') && + !isPrimitiveFilterValue(nestedValue) + ) { + return `Operator "${key}" at "${path.join('.')}" must use a primitive value.`; + } + if ((key === '$and' || key === '$or') && !Array.isArray(nestedValue)) { + return `Operator "${key}" at "${path.join('.')}" must use an array of filter objects.`; + } + + const nestedError = validateMetadataFilterValue(nestedValue, [...path, key]); + if (nestedError) { + return nestedError; + } + } + + return null; +} + +/** Validate a Pinecone metadata filter object; returns an error message or null if valid. */ +export function validateMetadataFilter(filter: Record): string | null { + for (const [field, value] of Object.entries(filter)) { + const error = validateMetadataFilterValue(value, [field]); + if (error) return error; + } + return null; +} diff --git a/src/server/namespace-router.ts b/src/server/namespace-router.ts new file mode 100644 index 0000000..5db484a --- /dev/null +++ b/src/server/namespace-router.ts @@ -0,0 +1,78 @@ +import type { NamespaceInfo } from './namespaces-cache.js'; + +export type RankedNamespace = { + namespace: string; + score: number; + record_count: number; + reasons: string[]; +}; + +/** + * Score a namespace for relevance to the query using only: + * - Query containing the namespace name (normalized) + * - Query containing any of the namespace's metadata field names + * No hardcoded namespace names or keyword lists; works for any index/namespace. + */ +function scoreNamespace( + query: string, + namespace: string, + fields: string[] +): { score: number; reasons: string[] } { + const q = query.toLowerCase(); + const name = namespace.toLowerCase(); + const reasons: string[] = []; + let score = 0; + + const normalizedName = name.replace(/[^a-z0-9]/g, ' ').trim(); + if (normalizedName && q.includes(normalizedName)) { + score += 3; + reasons.push('query mentions namespace name'); + } else { + const nameTokens = [...new Set(normalizedName.split(/\s+/).filter(Boolean))]; + for (const token of nameTokens) { + if (token.length >= 2 && q.includes(token)) { + score += 2; + reasons.push(`query matches namespace token: ${token}`); + } + } + } + + for (const field of fields) { + if (q.includes(field.toLowerCase())) { + score += 1; + reasons.push(`field hint: ${field}`); + } + } + + return { score, reasons: Array.from(new Set(reasons)) }; +} + +/** + * Rank namespaces by relevance to the query and return the top N. + * Uses name and metadata-field matching; on equal score, prefers smaller (more-specific) namespaces. + */ +export function rankNamespacesByQuery( + query: string, + namespaces: NamespaceInfo[], + topN: number +): RankedNamespace[] { + const limit = Number.isFinite(topN) ? Math.max(1, Math.floor(topN)) : 1; + return namespaces + .map((ns) => { + const fields = Object.keys(ns.metadata ?? {}); + const { score, reasons } = scoreNamespace(query.trim(), ns.namespace, fields); + return { + namespace: ns.namespace, + score, + record_count: ns.recordCount, + reasons, + }; + }) + .sort((a, b) => { + if (b.score !== a.score) return b.score - a.score; + // On equal score, prefer the smaller (more-specific) namespace so that + // targeted namespaces are chosen over large catch-all ones. + return a.record_count - b.record_count; + }) + .slice(0, limit); +} diff --git a/src/server/namespaces-cache.ts b/src/server/namespaces-cache.ts new file mode 100644 index 0000000..a40fe1c --- /dev/null +++ b/src/server/namespaces-cache.ts @@ -0,0 +1,42 @@ +import { FLOW_CACHE_TTL_MS } from '../constants.js'; +import { getPineconeClient } from './client-context.js'; + +export type NamespaceInfo = { + namespace: string; + recordCount: number; + metadata: Record; +}; + +type CacheEntry = { + data: NamespaceInfo[]; + expiresAt: number; +}; + +let namespacesCache: CacheEntry | null = null; + +/** Return namespace list with metadata; uses in-memory cache for FLOW_CACHE_TTL_MS. */ +export async function getNamespacesWithCache(): Promise<{ + data: NamespaceInfo[]; + cache_hit: boolean; + expires_at: number; +}> { + const now = Date.now(); + if (namespacesCache && now < namespacesCache.expiresAt) { + return { + data: namespacesCache.data, + cache_hit: true, + expires_at: namespacesCache.expiresAt, + }; + } + + const client = getPineconeClient(); + const data = await client.listNamespacesWithMetadata(); + const expiresAt = now + FLOW_CACHE_TTL_MS; + namespacesCache = { data, expiresAt }; + return { data, cache_hit: false, expires_at: expiresAt }; +} + +/** Clear the namespaces cache so the next call to getNamespacesWithCache refetches. */ +export function invalidateNamespacesCache(): void { + namespacesCache = null; +} diff --git a/src/server/query-suggestion.ts b/src/server/query-suggestion.ts new file mode 100644 index 0000000..f4a62d3 --- /dev/null +++ b/src/server/query-suggestion.ts @@ -0,0 +1,82 @@ +import { COUNT_FIELDS } from '../constants.js'; + +/** Suggested query params from namespace schema and user query. */ +export type SuggestQueryParamsResult = { + suggested_fields: string[]; + use_count_tool: boolean; + recommended_tool: 'count' | 'query_fast' | 'query_detailed'; + explanation: string; + namespace_found: boolean; +}; + +/** + * Suggests which fields to request and whether to use the count tool, + * based on the namespace's available metadata fields (from list_namespaces) and the user's natural language query. + */ +export function suggestQueryParams( + namespaceMetadataFields: Record | null, + userQuery: string +): SuggestQueryParamsResult { + const q = userQuery.toLowerCase().trim(); + const available = namespaceMetadataFields ? Object.keys(namespaceMetadataFields) : []; + const keepOnlyAvailable = (fields: string[]) => fields.filter((f) => available.includes(f)); + + if (!namespaceMetadataFields) { + return { + suggested_fields: [], + use_count_tool: false, + recommended_tool: 'query_fast', + explanation: + 'Namespace not found or has no metadata fields. Call list_namespaces first, then pass a valid namespace.', + namespace_found: false, + }; + } + if (available.length === 0) { + return { + suggested_fields: [], + use_count_tool: false, + recommended_tool: 'query_fast', + explanation: + 'Namespace has no metadata fields. Use list_namespaces to verify the namespace is correct.', + namespace_found: true, + }; + } + + // Count intent: "how many", "count", "number of", etc. + if (/\b(how many|count|number of|total number|paper count|documents? count)\b/.test(q)) { + const fields = keepOnlyAvailable([...COUNT_FIELDS]); + return { + suggested_fields: fields.length ? fields : available.slice(0, 5), + use_count_tool: true, + recommended_tool: 'count', + explanation: + 'User asked for a count. Use the count tool for this. If using query instead, use minimal fields (no chunk_text).', + namespace_found: true, + }; + } + + // Content intent: user wants to read/summarize content + if ( + /\b(content|summarize|summarise|what does|excerpt|text|say|details?|full text|body)\b/.test(q) + ) { + const fields = keepOnlyAvailable(['document_number', 'title', 'url', 'author', 'chunk_text']); + return { + suggested_fields: fields.length ? fields : available, + use_count_tool: false, + recommended_tool: 'query_detailed', + explanation: 'User asked for content or details; include chunk_text for snippets.', + namespace_found: true, + }; + } + + // List/browse intent: titles, links, list (minimal fields, no content) + const listFields = keepOnlyAvailable(['document_number', 'title', 'url', 'author']); + return { + suggested_fields: listFields.length ? listFields : available.slice(0, 5), + use_count_tool: false, + recommended_tool: 'query_fast', + explanation: + 'User asked for a list or browse; use minimal fields (no chunk_text) for smaller payload and cost.', + namespace_found: true, + }; +} diff --git a/src/server/reassemble-documents.test.ts b/src/server/reassemble-documents.test.ts new file mode 100644 index 0000000..33448eb --- /dev/null +++ b/src/server/reassemble-documents.test.ts @@ -0,0 +1,88 @@ +import { describe, it, expect } from 'vitest'; +import { reassembleByDocument } from './reassemble-documents.js'; +import type { SearchResult } from '../types.js'; + +describe('reassembleByDocument', () => { + it('groups chunks by document_number', () => { + const results: SearchResult[] = [ + { + id: 'c1', + content: 'First chunk.', + score: 0.9, + metadata: { document_number: 'P1234', chunk_index: 0 }, + reranked: false, + }, + { + id: 'c2', + content: 'Second chunk.', + score: 0.8, + metadata: { document_number: 'P1234', chunk_index: 1 }, + reranked: false, + }, + { + id: 'c3', + content: 'Other doc.', + score: 0.7, + metadata: { document_number: 'P5678' }, + reranked: false, + }, + ]; + const docs = reassembleByDocument(results); + expect(docs).toHaveLength(2); + const p1234 = docs.find((d) => d.document_id === 'P1234'); + const p5678 = docs.find((d) => d.document_id === 'P5678'); + expect(p1234?.merged_content).toBe('First chunk.\n\nSecond chunk.'); + expect(p1234?.chunk_count).toBe(2); + expect(p5678?.merged_content).toBe('Other doc.'); + expect(p5678?.chunk_count).toBe(1); + }); + + it('sorts chunks by chunk_index when present', () => { + const results: SearchResult[] = [ + { + id: 'b', + content: 'Second', + score: 0.5, + metadata: { document_number: 'D1', chunk_index: 1 }, + reranked: false, + }, + { + id: 'a', + content: 'First', + score: 0.9, + metadata: { document_number: 'D1', chunk_index: 0 }, + reranked: false, + }, + ]; + const docs = reassembleByDocument(results); + expect(docs[0].merged_content).toBe('First\n\nSecond'); + }); + + it('uses doc_id when document_number is missing', () => { + const results: SearchResult[] = [ + { + id: 'x', + content: 'Content', + score: 0.8, + metadata: { doc_id: 'my-doc-1' }, + reranked: false, + }, + ]; + const docs = reassembleByDocument(results); + expect(docs[0].document_id).toBe('my-doc-1'); + }); + + it('respects maxChunksPerDocument', () => { + const results: SearchResult[] = Array.from({ length: 10 }, (_, i) => ({ + id: `c${i}`, + content: `Chunk ${i}`, + score: 0.9 - i * 0.01, + metadata: { document_number: 'P1', chunk_index: i }, + reranked: false, + })); + const docs = reassembleByDocument(results, { maxChunksPerDocument: 3 }); + expect(docs).toHaveLength(1); + expect(docs[0].chunk_count).toBe(3); + expect(docs[0].merged_content).toBe('Chunk 0\n\nChunk 1\n\nChunk 2'); + }); +}); diff --git a/src/server/reassemble-documents.ts b/src/server/reassemble-documents.ts new file mode 100644 index 0000000..c66590e --- /dev/null +++ b/src/server/reassemble-documents.ts @@ -0,0 +1,113 @@ +/** + * Reassemble chunk-level search results into document-level results. + * Groups by document identity (document_number / doc_id / url) and merges chunk content + * for content analysis (summarization, full-document Q&A, etc.). + */ + +import type { PineconeMetadataValue, SearchResult } from '../types.js'; + +/** Default metadata keys tried for chunk ordering (RecursiveCharacterTextSplitter often adds these). */ +const CHUNK_ORDER_KEYS = ['chunk_index', 'start_index', 'loc'] as const; + +/** Derive a stable document key from hit metadata: document_number → url → doc_id → hit.id. */ +function getDocumentKey(hit: SearchResult): string { + const m = hit.metadata || {}; + const docNumber = m['document_number']; + const url = m['url']; + const docId = m['doc_id']; + return ( + (typeof docNumber === 'string' ? docNumber : undefined) ?? + (typeof url === 'string' ? url : undefined) ?? + (typeof docId === 'string' ? docId : undefined) ?? + hit.id ?? + '' + ); +} + +/** Return a numeric order for sorting chunks from metadata (chunk_index, start_index, or loc). */ +function getChunkOrder(metadata: Record): number { + for (const key of CHUNK_ORDER_KEYS) { + const v = metadata[key]; + if (typeof v === 'number' && Number.isFinite(v)) return v; + if (typeof v === 'string' && /^\d+$/.test(v)) return parseInt(v, 10); + } + return -1; +} + +export interface ReassembledDocument { + /** Document identity (document_number, doc_id, or url). */ + document_id: string; + /** Merged content from all chunks (ordered by chunk_index when available). */ + merged_content: string; + /** Metadata from the first chunk (title, author, url, etc.). */ + metadata: Record; + /** Number of chunks merged. */ + chunk_count: number; + /** Best score among chunks (for ranking documents). */ + best_score: number; +} + +/** + * Group search results by document and merge chunk content. + * Chunks are ordered by metadata chunk_index (or similar) when present; otherwise retrieval order. + * * + * Document identity is derived in priority order: document_number → url → doc_id → hit.id. + * Chunks whose metadata contains none of the first three keys are grouped by their raw vector ID, + * so each such chunk becomes its own single-chunk "document" in the output. + */ +export function reassembleByDocument( + results: SearchResult[], + options?: { + /** Max chunks to merge per document (avoids huge payloads). Default 200. */ + maxChunksPerDocument?: number; + /** Separator between chunk contents. Default double newline. */ + contentSeparator?: string; + } +): ReassembledDocument[] { + const maxChunks = Math.max(1, options?.maxChunksPerDocument ?? 200); + const separator = options?.contentSeparator ?? '\n\n'; + + const byDoc = new Map(); + + for (const hit of results) { + const key = getDocumentKey(hit); + if (!key) continue; + let list = byDoc.get(key); + if (!list) { + list = []; + byDoc.set(key, list); + } + list.push(hit); + } + + const out: ReassembledDocument[] = []; + + for (const [docId, chunks] of byDoc) { + const sorted = [...chunks].sort((a, b) => { + const orderA = getChunkOrder(a.metadata ?? {}); + const orderB = getChunkOrder(b.metadata ?? {}); + if (orderA >= 0 && orderB >= 0) return orderA - orderB; + if (orderA >= 0) return -1; + if (orderB >= 0) return 1; + return 0; + }); + + const toMerge = sorted.slice(0, maxChunks); + const merged_content = toMerge + .map((c) => c.content.trim()) + .filter(Boolean) + .join(separator); + const first = toMerge[0]; + const best_score = Math.max(...toMerge.map((c) => c.score)); + + out.push({ + document_id: docId, + merged_content, + metadata: (first?.metadata ?? {}) as Record, + chunk_count: toMerge.length, + best_score: Math.round(best_score * 10000) / 10000, + }); + } + + return out; +} diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts new file mode 100644 index 0000000..de67480 --- /dev/null +++ b/src/server/suggestion-flow.ts @@ -0,0 +1,64 @@ +import { FLOW_CACHE_TTL_MS } from '../constants.js'; + +type FlowState = { + updatedAt: number; + recommended_tool: 'count' | 'query_fast' | 'query_detailed'; + suggested_fields: string[]; + user_query: string; +}; + +const stateByNamespace = new Map(); + +/** + * Evict all entries older than FLOW_CACHE_TTL_MS. + * Called on every write so the map stays bounded without a background timer. + */ +function sweepExpired(): void { + const now = Date.now(); + for (const [ns, state] of stateByNamespace) { + if (now - state.updatedAt > FLOW_CACHE_TTL_MS) { + stateByNamespace.delete(ns); + } + } +} + +/** Record that suggest_query_params was called for this namespace (enables query/count for the flow). */ +export function markSuggested(namespace: string, state: Omit): void { + sweepExpired(); + stateByNamespace.set(namespace, { + ...state, + updatedAt: Date.now(), + }); +} + +/** Ensure suggest_query_params was called for this namespace within TTL; returns flow state or error. */ +export function requireSuggested(namespace: string): + | { + ok: true; + flow: FlowState; + } + | { + ok: false; + message: string; + } { + const state = stateByNamespace.get(namespace); + if (!state) { + return { + ok: false, + message: + 'Flow requires suggest_query_params first. Call suggest_query_params with namespace and user_query before query/count tools.', + }; + } + + const now = Date.now(); + if (now - state.updatedAt > FLOW_CACHE_TTL_MS) { + stateByNamespace.delete(namespace); + return { + ok: false, + message: + 'Previous suggest_query_params context expired (30 minutes). Call suggest_query_params again before query/count tools.', + }; + } + + return { ok: true, flow: state }; +} diff --git a/src/server/tool-error.ts b/src/server/tool-error.ts new file mode 100644 index 0000000..ec2274a --- /dev/null +++ b/src/server/tool-error.ts @@ -0,0 +1,16 @@ +/** + * Shared error handling for MCP tools: consistent logging and user-facing messages. + */ + +import { getLogLevel, error as logError } from '../logger.js'; + +/** User-facing error message: detailed in DEBUG, generic otherwise. */ +export function getToolErrorMessage(error: unknown, fallbackMessage: string): string { + const msg = error instanceof Error ? error.message : String(error); + return getLogLevel() === 'DEBUG' ? msg : fallbackMessage; +} + +/** Log tool failure to stderr via the level-based logger. */ +export function logToolError(toolName: string, error: unknown): void { + logError(`Error in ${toolName} tool`, error); +} diff --git a/src/server/tool-response.ts b/src/server/tool-response.ts new file mode 100644 index 0000000..bf56f29 --- /dev/null +++ b/src/server/tool-response.ts @@ -0,0 +1,29 @@ +export type TextPayload = { + content: Array<{ type: 'text'; text: string }>; + isError?: boolean; +}; + +/** Build an MCP tool success payload with JSON-stringified content. */ +export function jsonResponse(payload: unknown): TextPayload { + return { + content: [ + { + type: 'text', + text: JSON.stringify(payload, null, 2), + }, + ], + }; +} + +/** Build an MCP tool error payload with JSON-stringified content and isError: true. */ +export function jsonErrorResponse(payload: unknown): TextPayload { + return { + isError: true, + content: [ + { + type: 'text', + text: JSON.stringify(payload, null, 2), + }, + ], + }; +} diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts new file mode 100644 index 0000000..34ead93 --- /dev/null +++ b/src/server/tools/count-tool.ts @@ -0,0 +1,94 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getPineconeClient } from '../client-context.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { requireSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +const COUNT_RESPONSE_STATUS = 'success' as const; +type CountResponse = + | { + status: 'success'; + count: number; + truncated: boolean; + namespace: string; + metadata_filter?: Record; + } + | { status: 'error'; message: string }; + +/** Register the count tool on the MCP server. */ +export function registerCountTool(server: McpServer): void { + server.registerTool( + 'count', + { + description: + 'Get the number of unique documents matching a metadata filter and semantic query. ' + + 'Use when the user asks for a count (e.g. "how many papers by J. Lakos?", "John\'s paper count"). ' + + 'Uses semantic (dense) search only and requests only document identifiers (no content) for performance. ' + + 'Returns the number of unique documents (deduped by document_number/url/doc_id) up to 10,000; truncated=true if at least that many. ' + + 'Mandatory flow: call suggest_query_params first, then count. ' + + 'Use list_namespaces to discover namespace and metadata fields. ' + + 'For count-by-metadata only, use a broad query_text (e.g. "paper" or "document"). ' + + 'Same metadata_filter operators as query: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin.', + inputSchema: { + namespace: z + .string() + .describe('Namespace to count in. Use list_namespaces to discover namespaces.'), + query_text: z + .string() + .describe( + 'Search query text. Use a broad term (e.g. "paper", "document") when counting by metadata only.' + ), + metadata_filter: metadataFilterSchema + .optional() + .describe( + 'Optional metadata filter. Use exact field names from list_namespaces. E.g. {"author": {"$in": ["John Lakos", "J. Lakos"]}} for author count.' + ), + }, + }, + async (params) => { + try { + const { namespace, query_text, metadata_filter } = params; + if (!query_text.trim()) { + const response: CountResponse = { + status: 'error', + message: 'query_text cannot be empty', + }; + return jsonErrorResponse(response); + } + if (metadata_filter) { + const err = validateMetadataFilter(metadata_filter); + if (err) { + return jsonErrorResponse({ status: 'error', message: err }); + } + } + const flowCheck = requireSuggested(namespace); + if (!flowCheck.ok) { + return jsonErrorResponse({ status: 'error', message: flowCheck.message }); + } + const client = getPineconeClient(); + const { count, truncated } = await client.count({ + query: query_text.trim(), + namespace, + metadataFilter: metadata_filter, + }); + const response: CountResponse = { + status: COUNT_RESPONSE_STATUS, + count, + truncated, + namespace, + metadata_filter, + }; + return jsonResponse(response); + } catch (error) { + logToolError('count', error); + const response: CountResponse = { + status: 'error', + message: getToolErrorMessage(error, 'Failed to get count'), + }; + return jsonErrorResponse(response); + } + } + ); +} diff --git a/src/server/tools/generate-urls-tool.ts b/src/server/tools/generate-urls-tool.ts new file mode 100644 index 0000000..ea3adc2 --- /dev/null +++ b/src/server/tools/generate-urls-tool.ts @@ -0,0 +1,68 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { generateUrlForNamespace } from '../url-generation.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** Get metadata from a record (either record.metadata or the record itself). */ +function extractMetadata(record: Record): Record { + const nested = record['metadata']; + if (nested && typeof nested === 'object' && !Array.isArray(nested)) { + return nested as Record; + } + return record; +} + +/** Register the generate_urls tool on the MCP server. */ +export function registerGenerateUrlsTool(server: McpServer): void { + server.registerTool( + 'generate_urls', + { + description: + 'Generate URLs for retrieved results when metadata does not include url and URL is needed. ' + + 'Uses the URL generator registered for the given namespace (if any); returns unavailable for namespaces without a generator.', + inputSchema: { + namespace: z + .string() + .describe( + 'Target namespace. URL generation is supported only for namespaces that have a registered generator (call list_namespaces to discover namespaces).' + ), + records: z + .array(z.record(z.string(), z.unknown())) + .max(500) + .describe( + 'Array of records from retrieval results. Each item may be either metadata itself or an object containing a metadata field.' + ), + }, + }, + async (params) => { + try { + const { namespace, records } = params; + const results = records.map((record, index) => { + const metadata = extractMetadata(record); + const generated = generateUrlForNamespace(namespace, metadata); + return { + index, + url: generated.url, + method: generated.method, + reason: generated.reason ?? null, + metadata, + }; + }); + + return jsonResponse({ + status: 'success', + namespace, + count: results.length, + results, + }); + } catch (error) { + logToolError('generate_urls', error); + return jsonErrorResponse({ + status: 'error', + message: getToolErrorMessage(error, 'Failed to generate URLs'), + }); + } + } + ); +} diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts new file mode 100644 index 0000000..424df65 --- /dev/null +++ b/src/server/tools/guided-query-tool.ts @@ -0,0 +1,184 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { FAST_QUERY_FIELDS, MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; +import type { QueryResponse } from '../../types.js'; +import { getPineconeClient } from '../client-context.js'; +import { formatQueryResultRows } from '../format-query-result.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { rankNamespacesByQuery } from '../namespace-router.js'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { suggestQueryParams } from '../query-suggestion.js'; +import { markSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +type GuidedToolName = 'count' | 'query_fast' | 'query_detailed'; + +/** Register the guided_query orchestrator tool on the MCP server. */ +export function registerGuidedQueryTool(server: McpServer): void { + server.registerTool( + 'guided_query', + { + description: + 'Single orchestrator that runs routing + suggestion + execution in one call. ' + + 'Flow: optional namespace_router logic -> suggest_query_params logic -> executes count/query_fast/query_detailed. ' + + 'Returns decision_trace so behavior stays transparent and debuggable.', + inputSchema: { + user_query: z.string().describe('User question or intent.'), + namespace: z + .string() + .optional() + .describe( + 'Optional explicit namespace. If omitted, namespace_router logic will choose one.' + ), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to constrain results.'), + top_k: z + .number() + .int() + .min(MIN_TOP_K) + .max(MAX_TOP_K) + .default(10) + .describe('Result count for query_fast/query_detailed paths (1-100).'), + preferred_tool: z + .enum(['auto', 'count', 'query_fast', 'query_detailed']) + .default('auto') + .describe('Optional override. Use auto to follow suggestion logic.'), + enrich_urls: z + .boolean() + .default(true) + .describe( + 'If true, enrich result URLs using the namespace URL generator when metadata.url is missing (if supported for that namespace).' + ), + }, + }, + async (params) => { + try { + const { + user_query, + namespace: inputNamespace, + metadata_filter, + top_k, + preferred_tool, + enrich_urls, + } = params; + + if (!user_query?.trim()) { + return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); + } + + if (metadata_filter) { + const err = validateMetadataFilter(metadata_filter); + if (err) { + return jsonErrorResponse({ status: 'error', message: err }); + } + } + + const queryText = user_query.trim(); + const { data: namespaces, cache_hit } = await getNamespacesWithCache(); + const ranked = rankNamespacesByQuery(queryText, namespaces, 3); + + const namespace = inputNamespace ?? ranked[0]?.namespace; + if (!namespace) { + return jsonErrorResponse({ + status: 'error', + message: 'No namespace available. Please run list_namespaces and verify index data.', + }); + } + + const ns = namespaces.find((n) => n.namespace === namespace); + const suggestion = suggestQueryParams(ns?.metadata ?? null, queryText); + if (!suggestion.namespace_found) { + return jsonErrorResponse({ + status: 'error', + message: `Namespace "${namespace}" not found in cached namespaces. Call list_namespaces and retry.`, + }); + } + + const selectedTool: GuidedToolName = + preferred_tool === 'auto' ? suggestion.recommended_tool : preferred_tool; + markSuggested(namespace, { + recommended_tool: selectedTool, + suggested_fields: suggestion.suggested_fields, + user_query: queryText, + }); + + const client = getPineconeClient(); + const decision_trace = { + cache_hit, + input_namespace: inputNamespace ?? null, + routed_namespace: ranked[0]?.namespace ?? null, + selected_namespace: namespace, + ranked_namespaces: ranked, + suggested_fields: suggestion.suggested_fields, + suggested_tool: suggestion.recommended_tool, + selected_tool: selectedTool, + explanation: suggestion.explanation, + enrich_urls, + }; + + if (selectedTool === 'count') { + const { count, truncated } = await client.count({ + query: queryText, + namespace, + metadataFilter: metadata_filter, + }); + return jsonResponse({ + status: 'success', + decision_trace, + result: { + tool: 'count', + namespace, + query: queryText, + metadata_filter, + count, + truncated, + }, + }); + } + + const isFast = selectedTool === 'query_fast'; + const fields = + suggestion.suggested_fields.length > 0 + ? suggestion.suggested_fields + : isFast + ? [...FAST_QUERY_FIELDS] + : undefined; + const queryResults = await client.query({ + query: queryText, + namespace, + topK: top_k, + metadataFilter: metadata_filter, + useReranking: !isFast, + fields: fields?.length ? fields : undefined, + }); + const formattedResults = formatQueryResultRows(queryResults, { + namespace, + enrichUrls: enrich_urls, + }); + const result: QueryResponse = { + status: 'success', + mode: isFast ? 'query_fast' : 'query_detailed', + query: queryText, + namespace, + metadata_filter: metadata_filter, + result_count: formattedResults.length, + ...(fields?.length ? { fields } : {}), + results: formattedResults, + }; + return jsonResponse({ + status: 'success', + decision_trace, + result, + }); + } catch (error) { + logToolError('guided_query', error); + return jsonErrorResponse({ + status: 'error', + message: getToolErrorMessage(error, 'Failed to execute guided query'), + }); + } + } + ); +} diff --git a/src/server/tools/list-namespaces-tool.ts b/src/server/tools/list-namespaces-tool.ts new file mode 100644 index 0000000..57750fa --- /dev/null +++ b/src/server/tools/list-namespaces-tool.ts @@ -0,0 +1,47 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** Register the list_namespaces tool on the MCP server. */ +export function registerListNamespacesTool(server: McpServer): void { + server.registerTool( + 'list_namespaces', + { + description: + 'List all available namespaces in the Pinecone index with their metadata fields and record counts. ' + + 'Returns detailed information about each namespace including available metadata fields that can be used for filtering in queries. ' + + 'Use this tool first to discover which namespaces exist and what metadata fields are available for filtering. ' + + 'Results are cached in-memory for 30 minutes for better performance.', + inputSchema: {}, + }, + async () => { + try { + const { data: namespacesInfo, cache_hit, expires_at } = await getNamespacesWithCache(); + const now = Date.now(); + const ttlSeconds = Math.max(0, Math.floor((expires_at - now) / 1000)); + + const response = { + status: 'success', + cache_hit, + cache_ttl_seconds: ttlSeconds, + count: namespacesInfo.length, + namespaces: namespacesInfo.map((ns) => ({ + name: ns.namespace, + record_count: ns.recordCount, + metadata_fields: ns.metadata, + })), + }; + + return jsonResponse(response); + } catch (error) { + logToolError('list_namespaces', error); + const response = { + status: 'error', + message: getToolErrorMessage(error, 'Failed to list namespaces'), + }; + return jsonErrorResponse(response); + } + } + ); +} diff --git a/src/server/tools/namespace-router-tool.ts b/src/server/tools/namespace-router-tool.ts new file mode 100644 index 0000000..02e70d9 --- /dev/null +++ b/src/server/tools/namespace-router-tool.ts @@ -0,0 +1,55 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { rankNamespacesByQuery } from '../namespace-router.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** Register the namespace_router tool on the MCP server. */ +export function registerNamespaceRouterTool(server: McpServer): void { + server.registerTool( + 'namespace_router', + { + description: + 'Suggest likely namespace(s) for a user query using namespace names, metadata fields, and keyword heuristics. ' + + 'Use before suggest_query_params when namespace is unclear.', + inputSchema: { + user_query: z + .string() + .describe('User question/intent used to infer relevant namespace(s).'), + top_n: z + .number() + .int() + .min(1) + .max(5) + .default(3) + .describe('Maximum number of suggested namespaces (1-5).'), + }, + }, + async (params) => { + try { + const { user_query, top_n } = params; + if (!user_query?.trim()) { + return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); + } + const { data, cache_hit } = await getNamespacesWithCache(); + const ranked = rankNamespacesByQuery(user_query.trim(), data, top_n); + + const response = { + status: 'success' as const, + cache_hit, + user_query: user_query.trim(), + suggestions: ranked, + recommended_namespace: ranked[0]?.namespace ?? null, + }; + return jsonResponse(response); + } catch (error) { + logToolError('namespace_router', error); + return jsonErrorResponse({ + status: 'error', + message: getToolErrorMessage(error, 'Failed to route namespace'), + }); + } + } + ); +} diff --git a/src/server/tools/query-documents-tool.ts b/src/server/tools/query-documents-tool.ts new file mode 100644 index 0000000..798d91d --- /dev/null +++ b/src/server/tools/query-documents-tool.ts @@ -0,0 +1,133 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { + DEFAULT_QUERY_DOCUMENTS_TOP_K, + MAX_QUERY_DOCUMENTS_TOP_K, + QUERY_DOCUMENTS_MAX_CHUNKS, +} from '../../constants.js'; +import { getPineconeClient } from '../client-context.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { reassembleByDocument } from '../reassemble-documents.js'; +import { requireSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** + * Heuristic multiplier: chunks fetched = top_k × CHUNKS_PER_DOCUMENT, capped by + * QUERY_DOCUMENTS_MAX_CHUNKS. Set to 50 as a balance between recall and performance — + * documents with more than ~50 chunks may be truncated unless the caller passes a + * higher `max_chunks_per_document` (default 200, max 500). Increasing this constant + * raises Pinecone fetch latency and memory usage during reassembly. + */ +const CHUNKS_PER_DOCUMENT = 50; + +/** Register the query_documents tool (reassemble chunks into full documents) on the MCP server. */ +export function registerQueryDocumentsTool(server: McpServer): void { + server.registerTool( + 'query_documents', + { + description: + 'Run a semantic query and return whole documents (reassembled from chunks). ' + + 'Use for content analysis, summarization, or when you need full-document context. ' + + 'Chunks are grouped by document_number/doc_id/url, ordered by chunk_index when present (e.g. from RecursiveCharacterTextSplitter), and merged into one content per document. ' + + 'Mandatory flow: call suggest_query_params first. Use list_namespaces to discover namespaces.', + inputSchema: { + query_text: z.string().describe('Search query text. Be specific for better results.'), + namespace: z + .string() + .describe( + 'Namespace to search. Use list_namespaces/namespace_router first, then suggest_query_params.' + ), + top_k: z + .number() + .int() + .min(1) + .max(MAX_QUERY_DOCUMENTS_TOP_K) + .default(DEFAULT_QUERY_DOCUMENTS_TOP_K) + .describe( + `Number of documents to return (1-${MAX_QUERY_DOCUMENTS_TOP_K}). Each document is reassembled from its chunks. Default: ${DEFAULT_QUERY_DOCUMENTS_TOP_K}.` + ), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to narrow search.'), + max_chunks_per_document: z + .number() + .int() + .min(1) + .max(500) + .optional() + .describe( + 'Max chunks to merge per document (default 200). Lower for shorter merged_content.' + ), + }, + }, + async (params) => { + try { + const { + query_text, + namespace, + top_k = DEFAULT_QUERY_DOCUMENTS_TOP_K, + metadata_filter, + max_chunks_per_document, + } = params; + + if (!query_text?.trim()) { + return jsonErrorResponse({ + status: 'error', + message: 'query_text cannot be empty', + }); + } + + if (metadata_filter) { + const err = validateMetadataFilter(metadata_filter); + if (err) return jsonErrorResponse({ status: 'error', message: err }); + } + + const flowCheck = requireSuggested(namespace); + if (!flowCheck.ok) { + return jsonErrorResponse({ status: 'error', message: flowCheck.message }); + } + + const chunkLimit = Math.min(QUERY_DOCUMENTS_MAX_CHUNKS, top_k * CHUNKS_PER_DOCUMENT); + const client = getPineconeClient(); + const results = await client.query({ + query: query_text.trim(), + topK: chunkLimit, + namespace, + useReranking: true, + metadataFilter: metadata_filter, + fields: undefined, + }); + + const reassembled = reassembleByDocument(results, { + maxChunksPerDocument: max_chunks_per_document ?? 200, + }); + + const topDocuments = reassembled + .sort((a, b) => b.best_score - a.best_score) + .slice(0, top_k); + + return jsonResponse({ + status: 'success', + query: query_text.trim(), + namespace, + metadata_filter, + result_count: topDocuments.length, + documents: topDocuments.map((doc) => ({ + document_id: doc.document_id, + merged_content: doc.merged_content, + metadata: doc.metadata, + chunk_count: doc.chunk_count, + best_score: doc.best_score, + })), + }); + } catch (error) { + logToolError('query_documents', error); + return jsonErrorResponse({ + status: 'error', + message: getToolErrorMessage(error, 'Failed to query and reassemble documents'), + }); + } + } + ); +} diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts new file mode 100644 index 0000000..3797285 --- /dev/null +++ b/src/server/tools/query-tool.ts @@ -0,0 +1,179 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { FAST_QUERY_FIELDS, MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; +import type { QueryResponse } from '../../types.js'; +import { getPineconeClient } from '../client-context.js'; +import { formatQueryResultRows } from '../format-query-result.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { requireSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +type QueryMode = 'query' | 'query_fast' | 'query_detailed'; + +type QueryExecParams = { + query_text: string; + namespace: string; + top_k: number; + use_reranking: boolean; + metadata_filter?: Record; + fields?: string[]; + mode: QueryMode; +}; + +/** Run the query tool: validate flow, call Pinecone, format and return results. */ +async function executeQuery(params: QueryExecParams) { + const { query_text, namespace, top_k, use_reranking, metadata_filter, fields, mode } = params; + try { + if (!query_text.trim()) { + const response: QueryResponse = { + status: 'error', + message: 'Query text cannot be empty', + }; + return jsonErrorResponse(response); + } + + if (metadata_filter) { + const filterValidationError = validateMetadataFilter(metadata_filter); + if (filterValidationError) { + const response: QueryResponse = { + status: 'error', + message: filterValidationError, + }; + return jsonErrorResponse(response); + } + } + + const flowCheck = requireSuggested(namespace); + if (!flowCheck.ok) { + return jsonErrorResponse({ status: 'error', message: flowCheck.message }); + } + + const client = getPineconeClient(); + const results = await client.query({ + query: query_text.trim(), + topK: top_k, + namespace, + useReranking: use_reranking, + metadataFilter: metadata_filter, + fields: fields?.length ? fields : undefined, + }); + + const formattedResults = formatQueryResultRows(results); + + const response: QueryResponse = { + status: 'success', + mode, + query: query_text, + namespace, + metadata_filter: metadata_filter, + result_count: formattedResults.length, + results: formattedResults, + ...(fields?.length ? { fields } : {}), + }; + return jsonResponse(response); + } catch (error) { + logToolError(mode, error); + const response: QueryResponse = { + status: 'error', + message: getToolErrorMessage(error, 'An error occurred while processing your query'), + }; + return jsonErrorResponse(response); + } +} + +const baseSchema = { + query_text: z.string().describe('Search query text. Be specific for better results.'), + namespace: z + .string() + .describe( + 'Namespace to search within. Use list_namespaces/namespace_router first, then suggest_query_params before querying.' + ), + top_k: z + .number() + .int() + .min(MIN_TOP_K) + .max(MAX_TOP_K) + .default(10) + .describe('Number of results to return (1-100). Default: 10'), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to narrow down search results.'), + fields: z + .array(z.string()) + .optional() + .describe( + 'Optional field names to return from Pinecone. Use suggest_query_params suggested_fields for better performance.' + ), +}; + +/** Register the unified query tool (query_fast / query_detailed) on the MCP server. */ +export function registerQueryTool(server: McpServer): void { + server.registerTool( + 'query', + { + description: + 'Full query tool with optional reranking. Mandatory flow: call suggest_query_params first. ' + + 'For lighter retrieval use query_fast; for content-heavy retrieval use query_detailed.', + inputSchema: { + ...baseSchema, + use_reranking: z + .boolean() + .default(true) + .describe( + 'Whether to use semantic reranking for better relevance. Slower but more accurate.' + ), + }, + }, + async (params) => + executeQuery({ + ...params, + top_k: params.top_k, + use_reranking: params.use_reranking, + mode: 'query', + }) + ); + + server.registerTool( + 'query_fast', + { + description: + 'Fast query preset. Mandatory flow: call suggest_query_params first. ' + + 'Defaults to no reranking and lightweight fields for lower latency/cost.', + inputSchema: { + ...baseSchema, + }, + }, + async (params) => + executeQuery({ + ...params, + top_k: params.top_k, + use_reranking: false, + fields: params.fields?.length ? params.fields : [...FAST_QUERY_FIELDS], + mode: 'query_fast', + }) + ); + + server.registerTool( + 'query_detailed', + { + description: + 'Detailed query preset. Mandatory flow: call suggest_query_params first. ' + + 'Designed for reading/summarization workflows with content snippets.', + inputSchema: { + ...baseSchema, + use_reranking: z + .boolean() + .default(true) + .describe('Whether to use semantic reranking for better precision (default true).'), + }, + }, + async (params) => + executeQuery({ + ...params, + top_k: params.top_k ?? 10, + use_reranking: params.use_reranking ?? true, + mode: 'query_detailed', + }) + ); +} diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts new file mode 100644 index 0000000..b7b0ec7 --- /dev/null +++ b/src/server/tools/suggest-query-params-tool.ts @@ -0,0 +1,64 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { getNamespacesWithCache } from '../namespaces-cache.js'; +import { suggestQueryParams } from '../query-suggestion.js'; +import { markSuggested } from '../suggestion-flow.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** Register the suggest_query_params tool on the MCP server. */ +export function registerSuggestQueryParamsTool(server: McpServer): void { + server.registerTool( + 'suggest_query_params', + { + description: + "Suggest which fields to request and whether to use the count tool, based on the namespace schema (from list_namespaces) and the user's natural language query. " + + 'Call list_namespaces first to get available namespaces and metadata fields. Then call this tool with the target namespace and the user query; ' + + 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count/query_fast/query_detailed), and an explanation. ' + + 'This step is mandatory before query/count tools; use the returned suggested_fields in query tools to reduce payload and cost.', + inputSchema: { + namespace: z + .string() + .describe( + 'Namespace to query. Must match a name from list_namespaces so the tool can look up available metadata fields.' + ), + user_query: z + .string() + .describe( + 'The user\'s natural language question or intent (e.g. "list papers by Lakos with titles and links", "how many papers by Wong?", "what do the contracts papers say?").' + ), + }, + }, + async (params) => { + try { + const { namespace, user_query } = params; + if (!user_query?.trim()) { + return jsonErrorResponse({ status: 'error', message: 'user_query cannot be empty' }); + } + const { data: namespacesInfo, cache_hit } = await getNamespacesWithCache(); + const ns = namespacesInfo.find((n) => n.namespace === namespace); + const metadataFields = ns?.metadata ?? null; + const result = suggestQueryParams(metadataFields, user_query.trim()); + if (result.namespace_found) { + markSuggested(namespace, { + recommended_tool: result.recommended_tool, + suggested_fields: result.suggested_fields, + user_query: user_query.trim(), + }); + } + const response = { + ...result, + status: 'success' as const, + cache_hit, + }; + return jsonResponse(response); + } catch (error) { + logToolError('suggest_query_params', error); + return jsonErrorResponse({ + status: 'error', + message: getToolErrorMessage(error, 'Failed to suggest query params'), + }); + } + } + ); +} diff --git a/src/server/url-generation.test.ts b/src/server/url-generation.test.ts new file mode 100644 index 0000000..e1d7616 --- /dev/null +++ b/src/server/url-generation.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from 'vitest'; +import { generateUrlForNamespace } from './url-generation.js'; + +describe('generateUrlForNamespace', () => { + it('uses existing metadata.url when present', () => { + const r = generateUrlForNamespace('mailing', { + url: 'https://example.com/custom', + doc_id: 'ignored', + }); + expect(r.url).toBe('https://example.com/custom'); + expect(r.method).toBe('metadata.url'); + }); + + it('generates mailing URL from doc_id', () => { + const r = generateUrlForNamespace('mailing', { + doc_id: 'boost-announce@lists.boost.org/message/O5VYCDZADVDHK5Z5LAYJBHMDOAFQL7P6', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-announce@lists.boost.org/message/O5VYCDZADVDHK5Z5LAYJBHMDOAFQL7P6/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('generates mailing URL from thread_id when doc_id missing', () => { + const r = generateUrlForNamespace('mailing', { + thread_id: 'boost@lists.boost.org/thread/ABC123', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost@lists.boost.org/thread/ABC123/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('generates mailing URL as list_name/message/doc_id when list_name present and doc_id does not contain it', () => { + const r = generateUrlForNamespace('mailing', { + list_name: 'boost-users', + doc_id: '12345', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-users/message/12345/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('uses msg_id when list_name present and doc_id missing', () => { + const r = generateUrlForNamespace('mailing', { + list_name: 'boost-announce', + msg_id: '67890', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-announce/message/67890/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('uses single-path form when doc_id contains list_name (no list_name/message split)', () => { + const r = generateUrlForNamespace('mailing', { + list_name: 'boost-users', + doc_id: 'boost-users@lists.boost.org/message/12345', + }); + expect(r.url).toBe( + 'https://lists.boost.org/archives/list/boost-users@lists.boost.org/message/12345/' + ); + expect(r.method).toBe('generated.mailing'); + }); + + it('uses slack source when available', () => { + const r = generateUrlForNamespace('slack-Cpplang', { + source: 'https://app.slack.com/client/T123/C123/p123', + team_id: 'T999', + channel_id: 'C999', + doc_id: '1.2', + }); + expect(r.url).toBe('https://app.slack.com/client/T123/C123/p123'); + expect(r.method).toBe('metadata.source'); + }); + + it('generates slack URL from team/channel/doc_id', () => { + const r = generateUrlForNamespace('slack-Cpplang', { + team_id: 'T123456789', + channel_id: 'C123456', + doc_id: '1234567.890', + }); + expect(r.url).toBe('https://app.slack.com/client/T123456789/C123456/p1234567890'); + expect(r.method).toBe('generated.slack'); + }); + + it('returns unavailable for unsupported namespace', () => { + const r = generateUrlForNamespace('wg21-papers', { doc_id: 'x' }); + expect(r.url).toBeNull(); + expect(r.method).toBe('unavailable'); + }); + it('returns unavailable for mailing when no doc_id or thread_id', () => { + const r = generateUrlForNamespace('mailing', { author: 'someone' }); + expect(r.url).toBeNull(); + expect(r.method).toBe('unavailable'); + }); + + it('returns unavailable for slack-Cpplang when required fields are missing', () => { + const r = generateUrlForNamespace('slack-Cpplang', { + team_id: 'T123', + // channel_id missing, doc_id missing, no source + }); + expect(r.url).toBeNull(); + expect(r.method).toBe('unavailable'); + }); +}); diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts new file mode 100644 index 0000000..32dd82a --- /dev/null +++ b/src/server/url-generation.ts @@ -0,0 +1,104 @@ +export type UrlGenerationResult = { + url: string | null; + method: + | 'metadata.url' + | 'metadata.source' + | 'generated.mailing' + | 'generated.slack' + | 'unavailable'; + reason?: string; +}; + +type UrlGenerator = (metadata: Record) => UrlGenerationResult; + +/** Registry of namespace -> URL generator. Built-ins are registered below; more can be added at runtime. */ +const urlGenerators = new Map(); + +/** Return a trimmed non-empty string or null for empty/missing values. */ +function asString(value: unknown): string | null { + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : null; +} + +/** + * Build a mailing-list URL (e.g. Boost archives). + * Two cases: + * 1. If metadata has list_name and doc_id (or msg_id) and the message id does not contain list_name, + * URL is https://lists.boost.org/archives/list/{list_name}/message/{doc_id}/ + * 2. Otherwise use doc_id or thread_id as the list path: .../list/{doc_id_or_thread_id}/ + */ +function generatorMailing(metadata: Record): UrlGenerationResult { + const listName = asString(metadata['list_name']); + const docId = asString(metadata['doc_id']) ?? asString(metadata['msg_id']); + const threadId = asString(metadata['thread_id']); + + if (listName && docId && !docId.includes(listName)) { + return { + url: `https://lists.boost.org/archives/list/${listName}/message/${docId}/`, + method: 'generated.mailing', + }; + } + + const docIdOrThread = docId ?? threadId; + if (!docIdOrThread) { + return { + url: null, + method: 'unavailable', + reason: 'mailing requires doc_id, msg_id, or thread_id to generate URL', + }; + } + return { + url: `https://lists.boost.org/archives/list/${docIdOrThread}/`, + method: 'generated.mailing', + }; +} + +/** Build a Slack message URL from source or team_id/channel_id/doc_id. */ +function generatorSlackCpplang(metadata: Record): UrlGenerationResult { + const source = asString(metadata['source']); + if (source) { + return { url: source, method: 'metadata.source' }; + } + const teamId = asString(metadata['team_id']); + const channelId = asString(metadata['channel_id']); + const docId = asString(metadata['doc_id']); + if (!teamId || !channelId || !docId) { + return { + url: null, + method: 'unavailable', + reason: 'slack-Cpplang requires team_id, channel_id, and doc_id (or source)', + }; + } + const messageId = docId.replace(/\./g, ''); + return { + url: `https://app.slack.com/client/${teamId}/${channelId}/p${messageId}`, + method: 'generated.slack', + }; +} + +urlGenerators.set('mailing', generatorMailing); +urlGenerators.set('slack-Cpplang', generatorSlackCpplang); + +/** + * Generate a URL for a record in the given namespace when metadata.url is missing. + * Uses the registry of URL generators; returns unavailable for namespaces without a generator. + */ +export function generateUrlForNamespace( + namespace: string, + metadata: Record +): UrlGenerationResult { + const existingUrl = asString(metadata['url']); + if (existingUrl) { + return { url: existingUrl, method: 'metadata.url' }; + } + + const generator = urlGenerators.get(namespace); + if (generator) { + return generator(metadata); + } + + return { + url: null, + method: 'unavailable', + reason: `URL generation is not supported for namespace "${namespace}"`, + }; +} diff --git a/src/types.ts b/src/types.ts index b29ff84..485e8a1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -2,6 +2,9 @@ * Types for Pinecone Read-Only MCP */ +/** Pinecone metadata value types: string, number, boolean, or list of strings */ +export type PineconeMetadataValue = string | number | boolean | string[]; + export interface PineconeClientConfig { apiKey: string; indexName?: string; @@ -13,14 +16,14 @@ export interface SearchResult { id: string; content: string; score: number; - metadata: Record; + metadata: Record; reranked: boolean; } export interface PineconeHit { _id: string; _score: number; - fields: Record; + fields: Record; } export interface PineconeSearchResponse { @@ -30,15 +33,30 @@ export interface PineconeSearchResponse { } export interface NamespaceStats { - namespaces?: Record; + namespaces?: Record; } export interface QueryParams { query: string; topK?: number; namespace: string; - metadataFilter?: Record; + metadataFilter?: Record; useReranking?: boolean; + /** If set, only these fields are requested from Pinecone (e.g. ["document_number", "title", "url"]). Omit for all fields. Include "chunk_text" for content. */ + fields?: string[]; +} + +/** Parameters for count-only requests (high top_k, no reranking). */ +export interface CountParams { + query: string; + namespace: string; + metadataFilter?: Record; +} + +/** Result of a count request: unique document count (deduped by doc id/url); truncated when at least COUNT_TOP_K. */ +export interface CountResult { + count: number; + truncated: boolean; } export interface ListNamespacesResponse { @@ -50,10 +68,13 @@ export interface ListNamespacesResponse { export interface QueryResponse { status: 'success' | 'error'; + mode?: 'query' | 'query_fast' | 'query_detailed'; query?: string; namespace?: string; - metadata_filter?: Record; + metadata_filter?: Record; result_count?: number; + /** Present when the query requested specific fields. */ + fields?: string[]; results?: Array<{ paper_number: string | null; title: string; @@ -62,7 +83,52 @@ export interface QueryResponse { content: string; score: number; reranked: boolean; - metadata?: Record; // Include all metadata fields + metadata?: Record; }>; message?: string; } + +/** Internal merged hit shape before rerank (dense + sparse deduped). */ +export interface MergedHit { + _id: string; + _score: number; + chunk_text: string; + metadata: Record; +} + +/** + * Handle for a specific namespace, returned by SearchableIndex.namespace(). + * Carries the legacy query() path (vector-based metadata sampling) and the + * backward-compatible searchRecords() fallback. + */ +export interface NamespaceHandle { + query?(opts: { topK: number; vector: number[]; includeMetadata: boolean }): Promise<{ + matches?: Array<{ metadata?: Record }>; + }>; + searchRecords?(params: { + query: Record; + }): Promise<{ result?: { hits?: PineconeHit[] } }>; +} + +/** + * Minimal top-level index interface for hybrid search (dense/sparse) and namespace discovery. + * Methods are optional because the object is obtained via an `as unknown as` cast from the + * Pinecone SDK, whose concrete shape can vary across SDK versions. + */ +export interface SearchableIndex { + describeIndexStats?(): Promise<{ + dimension?: number; + namespaces?: Record; + }>; + search?(opts: { + namespace?: string; + query: Record; + fields?: string[]; + }): Promise<{ result?: { hits?: PineconeHit[] } }>; + /** Return a namespace-scoped handle for metadata sampling or legacy record queries. */ + namespace?(name: string): NamespaceHandle; + /** Backward-compatible fallback when the SDK exposes searchRecords on the top-level index. */ + searchRecords?(params: { + query: Record; + }): Promise<{ result?: { hits?: PineconeHit[] } }>; +}