From 74ff525adcc282457a4b8ff5afa5ad063790f761 Mon Sep 17 00:00:00 2001 From: zho Date: Wed, 6 May 2026 02:06:31 +0800 Subject: [PATCH 01/12] addressed review-library.md --- .env.example | 19 +- .github/dependabot.yml | 15 ++ .github/workflows/ci.yml | 31 ++- .npmignore | 1 + CHANGELOG.md | 48 ++-- README.md | 46 +++- package-lock.json | 241 ++++++++++++++++++ package.json | 16 +- src/config.ts | 135 +++++++++- src/constants.ts | 24 +- src/index.ts | 185 +++++--------- src/logger.ts | 90 ++++++- src/pinecone-client.ts | 165 +++++++++--- src/server.ts | 65 ++++- src/server/format-query-result.ts | 41 ++- src/server/namespaces-cache.ts | 10 +- src/server/query-suggestion.ts | 2 +- src/server/suggestion-flow.ts | 38 ++- src/server/tools/count-tool.ts | 10 +- src/server/tools/keyword-search-tool.ts | 3 + src/server/tools/list-namespaces-tool.ts | 1 + src/server/tools/suggest-query-params-tool.ts | 2 +- src/server/url-generation.ts | 51 +++- src/types.ts | 52 +++- tsconfig.json | 2 +- vitest.config.ts | 21 +- 26 files changed, 1063 insertions(+), 251 deletions(-) diff --git a/.env.example b/.env.example index a803b2b..4cb373c 100644 --- a/.env.example +++ b/.env.example @@ -13,5 +13,22 @@ PINECONE_RERANK_MODEL=bge-reranker-v2-m3 PINECONE_TOP_K=10 # Optional: Logging level (default: INFO) -# Options: DEBUG, INFO, WARNING, ERROR, CRITICAL +# Options: DEBUG, INFO, WARN, ERROR PINECONE_READ_ONLY_MCP_LOG_LEVEL=INFO + +# Optional: Logging format (default: text) +# Options: text, json +PINECONE_READ_ONLY_MCP_LOG_FORMAT=text + +# Optional: Sparse index name (default: ${PINECONE_INDEX_NAME}-sparse) +# PINECONE_SPARSE_INDEX_NAME=rag-hybrid-sparse + +# Optional: Namespace + suggestion-flow cache TTL in seconds (default: 1800 = 30min) +# PINECONE_CACHE_TTL_SECONDS=1800 + +# Optional: Disable the suggest_query_params flow gate (default: false). +# WARNING: turning this on bypasses the safety guard. A WARN log fires at startup. +# PINECONE_DISABLE_SUGGEST_FLOW=false + +# Optional: Request timeout for Pinecone calls, in milliseconds (default: 15000) +# PINECONE_REQUEST_TIMEOUT_MS=15000 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 43c92d3..99a41bc 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,6 +10,21 @@ updates: labels: - "dependencies" - "automated" + groups: + vitest: + patterns: + - "vitest" + - "@vitest/*" + typescript-eslint: + patterns: + - "typescript-eslint" + - "@typescript-eslint/*" + eslint-prettier: + patterns: + - "eslint" + - "@eslint/*" + - "prettier" + - "eslint-config-prettier" - package-ecosystem: "github-actions" directory: "/" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0d97076..88c2319 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,13 +5,15 @@ on: branches: [main] push: branches: [main] - workflow_call: # Allow this workflow to be called by other workflows + workflow_call: jobs: build-and-test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: + os: [ubuntu-latest, windows-latest, macos-latest] node-version: [18.x, 20.x, 22.x] steps: @@ -41,9 +43,32 @@ jobs: - name: Smoke test CLI run: node dist/index.js --help - - name: Run tests + - name: Run tests with coverage + if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' + run: npm run test:coverage + + - name: Run tests (no coverage) + if: "!(matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x')" run: npm test + - name: Generate CycloneDX SBOM (lockfile-based) + if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' + run: npx --yes @cyclonedx/cyclonedx-npm --package-lock-only --output-file sbom.cdx.json + + - name: Upload SBOM artifact + if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' + uses: actions/upload-artifact@v4 + with: + name: sbom-cyclonedx + path: sbom.cdx.json + + - name: Upload coverage to Codecov + if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' + uses: codecov/codecov-action@v5 + with: + files: ./coverage/lcov.info + fail_ci_if_error: false + quality: runs-on: ubuntu-latest steps: diff --git a/.npmignore b/.npmignore index 8486c95..2f71f50 100644 --- a/.npmignore +++ b/.npmignore @@ -32,6 +32,7 @@ node_modules/ .vscode/ .idea/ dist/ +docs/ # Testing coverage/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 37f4a3d..11398e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,38 +4,43 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +Future releases are managed automatically by [release-please](https://github.com/googleapis/release-please). ## [Unreleased] ### Added -- `list_namespaces` tool: discovers all namespaces with record counts and sampled metadata fields -- `namespace_router` tool: ranks namespaces by relevance to a natural-language query -- `suggest_query_params` tool: mandatory flow gate that recommends fields and tool variant before queries -- `count` tool: counts unique documents matching a query, with `truncated` flag when results exceed `COUNT_TOP_K` -- `query_fast` tool: lightweight chunk-level search with minimal field set -- `query_detailed` tool: chunk-level search with optional semantic reranking -- `query` tool: unified query entry-point supporting `query_fast` and `query_detailed` modes -- `guided_query` tool: single-call orchestrator that runs routing → suggestion → execution, returning a `decision_trace` -- `generate_urls` tool: synthesizes URLs for records whose metadata lacks a `url` field -- `query_documents` tool: reassembles full documents from chunks grouped by document identifier -- Centralized structured logger (`src/logger.ts`) with level-gated stderr output and stack-trace capture -- Dockerfile for containerised deployment -- `About.md` project documentation covering architecture, tools, and operating principles +- `SERVER_VERSION` is now read from `package.json` at runtime so MCP `serverInfo` always matches the published package version. +- `--version` CLI flag prints the package version and exits. +- `list_namespaces` response now includes `expires_at_iso` so clients see the cache expiry as an ISO-8601 timestamp without converting `cache_ttl_seconds`. +- `docs/` handbook: `TOOLS.md`, `CONFIGURATION.md`, `FAQ.md`, `MIGRATION.md`, `CI_CD.md`, and `docs/README.md` index; root `RELEASING.md` stub points to `docs/RELEASING.md`. +- `SECURITY.md`, `CONTRIBUTING.md`, and `CODE_OF_CONDUCT.md` for OSS hygiene. +- `examples/README.md` describing the library embedding sample. +- GitHub Actions: **multi-OS** CI matrix (Ubuntu, Windows, macOS × Node 18/20/22), **Codecov** upload (Ubuntu + Node 20), **CycloneDX SBOM** artifact, **Release Please**, and **Docker** multi-arch publish to **GHCR**. +- `src/config.test.ts` and `npm run test:coverage` with Vitest coverage thresholds (see `vitest.config.ts`). +- `@vitest/coverage-v8` devDependency for coverage reports (`lcov`, `json-summary`, HTML). ### Changed -- Modularised server into focused files under `src/server/` (tools, caches, formatters, suggestion flow) -- CI workflow updated: multi-node matrix (`18.x`, `20.x`, `22.x`), separate quality job with `continue-on-error` -- Replaced all `console.error` calls in `pinecone-client.ts` with typed `logInfo` / `logDebug` / `logError` +- `.env.example` log-level options corrected to the four levels actually supported (`DEBUG`, `INFO`, `WARN`, `ERROR`); the stale `WARNING`/`CRITICAL` values are gone. +- README Slack URL example now matches the generator output (`https://app.slack.com/client/{team_id}/{channel_id}/p{messageId}`). +- README "Comparison with Python Version" no longer claims an identical API interface; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. +- `npm run ci` now runs `test:coverage` so merges are gated on coverage thresholds. +- Dependabot groups related **vitest**, **typescript-eslint**, and **eslint/prettier** updates. -### Fixed -- Timestamp-based metadata filter now correctly handles numeric epoch values +### Removed + +- Dead `test:mcp` npm script (referenced a `test-mcp-server.js` file that has never existed). -### Security +## [0.1.6] - 2026-04-24 -- N/A +Historical 0.1.x releases (0.1.0 → 0.1.6) shipped the full tool surface +(`list_namespaces`, `namespace_router`, `suggest_query_params`, `count`, +`query`, `query_fast`, `query_detailed`, `keyword_search`, `query_documents`, +`guided_query`, `generate_urls`), the structured `src/logger.ts`, the +`Dockerfile`, and the modularised `src/server/` layout. See git history for +details — going forward, all changes are tracked here by release-please. ## [0.1.1] - 2026-01-27 @@ -69,6 +74,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Environment variable support - Full documentation and examples -[Unreleased]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.1...HEAD +[Unreleased]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.6...HEAD +[0.1.6]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.1...v0.1.6 [0.1.1]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/compare/v0.1.0...v0.1.1 [0.1.0]: https://github.com/CppDigest/pinecone-read-only-mcp-typescript/releases/tag/v0.1.0 diff --git a/README.md b/README.md index c5169c4..64ca28d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,20 @@ A Model Context Protocol (MCP) server that provides semantic search over Pinecone vector databases using hybrid search (dense + sparse) with reranking. +## Documentation + +| Doc | Description | +|-----|---------------| +| [docs/README.md](docs/README.md) | Index of all guides | +| [docs/TOOLS.md](docs/TOOLS.md) | Tool catalog & flows | +| [docs/CONFIGURATION.md](docs/CONFIGURATION.md) | Env vars, CLI flags, library config | +| [docs/FAQ.md](docs/FAQ.md) | Common questions | +| [docs/MIGRATION.md](docs/MIGRATION.md) | Deprecations & breaking changes | +| [docs/CI_CD.md](docs/CI_CD.md) | GitHub Actions, SBOM, Docker, releases | +| [RELEASING.md](RELEASING.md) | Pointer to the full release guide in `docs/` | +| [CONTRIBUTING.md](CONTRIBUTING.md) | How to contribute | +| [SECURITY.md](SECURITY.md) | Vulnerability reporting | + ## Features - **Hybrid Search**: Combines dense and sparse embeddings for superior recall @@ -54,16 +68,19 @@ npm run build ## Configuration -The server requires a Pinecone API key and supports the following configuration options: +You need a **Pinecone API key** and (by default) a **dense** index plus matching **sparse** index; see [docs/CONFIGURATION.md](docs/CONFIGURATION.md) for every environment variable and CLI flag. + +Quick reference: -### Environment Variables +| Variable | Required | Default | +| -------- | -------- | ------- | +| `PINECONE_API_KEY` | Yes (for live Pinecone) | — | +| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | +| `PINECONE_SPARSE_INDEX_NAME` | No | `{index}-sparse` | +| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` (`DEBUG`–`ERROR`) | +| `PINECONE_READ_ONLY_MCP_LOG_FORMAT` | No | `text` (`json` for log pipelines) | -| Variable | Required | Default | Description | -| ---------------------------------- | -------- | ---------------------- | ------------------------------------------------ | -| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | -| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name (dense + sparse for hybrid) | -| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | -| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | +Run `pinecone-read-only-mcp --help` for CLI equivalents (`--cache-ttl-seconds`, `--request-timeout-ms`, `--disable-suggest-flow`, etc.). ### Claude Desktop Configuration @@ -301,7 +318,7 @@ Rules: Format: `https://lists.boost.org/archives/list/{doc_id_or_thread_id}/` - **`slack-Cpplang`**: prefer `source` directly if present; otherwise use `team_id`, `channel_id`, and `doc_id` `message_id = doc_id.replace('.', '')` - Format: `https://app.slack.com/{team_id}/{channel_id}/p{message_id}` + Format: `https://app.slack.com/client/{team_id}/{channel_id}/p{message_id}` **Parameters:** @@ -577,13 +594,20 @@ npm run dev -- --api-key YOUR_API_KEY ## Comparison with Python Version -This TypeScript implementation provides the same functionality as the [Python version](https://github.com/CppDigest/pinecone-read-only-mcp) with the following benefits: +This TypeScript implementation grew out of the [Python version](https://github.com/CppDigest/pinecone-read-only-mcp) and now exposes a strict superset of its tool surface, including: + +- `guided_query` (single-call orchestrator with decision trace) +- `query_documents` (full-document reassembly from chunks) +- `keyword_search` (sparse-index-only retrieval) +- `namespace_router` and `suggest_query_params` (flow guidance) +- `count` and `generate_urls` + +Other benefits: - Native Node.js integration - Better npm ecosystem integration - TypeScript type safety - Similar performance characteristics -- Same API interface ## Troubleshooting diff --git a/package-lock.json b/package-lock.json index b47737f..b3865b8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@types/chai": "^5.2.3", "@types/deep-eql": "^4.0.2", "@types/node": "^25.0.10", + "@vitest/coverage-v8": "^4.0.18", "eslint": "^10.0.2", "eslint-config-prettier": "^10.1.8", "prettier": "^3.5.3", @@ -34,6 +35,66 @@ "node": ">=18.0.0" } }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.3", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.3.tgz", + "integrity": "sha512-b3ctpQwp+PROvU/cttc4OYl4MzfJUWy6FZg+PMXfzmt/+39iHVF0sDfqay8TQM3JA2EUOyKcFZt75jWriQijsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.0" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", + "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.2.tgz", @@ -660,6 +721,16 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.5.5", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", @@ -667,6 +738,17 @@ "dev": true, "license": "MIT" }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, "node_modules/@modelcontextprotocol/sdk": { "version": "1.29.0", "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.29.0.tgz", @@ -1118,6 +1200,7 @@ "integrity": "sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.19.0" } @@ -1167,6 +1250,7 @@ "integrity": "sha512-TI1XGwKbDpo9tRW8UDIXCOeLk55qe9ZFGs8MTKU6/M08HWTw52DD/IYhfQtOEhEdPhLMT26Ka/x7p70nd3dzDg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.59.0", "@typescript-eslint/types": "8.59.0", @@ -1352,6 +1436,37 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/@vitest/coverage-v8": { + "version": "4.0.18", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.0.18.tgz", + "integrity": "sha512-7i+N2i0+ME+2JFZhfuz7Tg/FqKtilHjGyGvoHYQ6iLV0zahbsJ9sljC9OcFcPDbhYKCet+sG8SsVqlyGvPflZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.0.18", + "ast-v8-to-istanbul": "^0.3.10", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.1", + "obug": "^2.1.1", + "std-env": "^3.10.0", + "tinyrainbow": "^3.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.0.18", + "vitest": "4.0.18" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, "node_modules/@vitest/expect": { "version": "4.0.18", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.0.18.tgz", @@ -1482,6 +1597,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -1542,6 +1658,18 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "0.3.12", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-0.3.12.tgz", + "integrity": "sha512-BRRC8VRZY2R4Z4lFIL35MwNXmwVqBityvOIwETtsCSwvjl0IdgFsy9NhdaA6j74nUdtJJlIypeRhpDam19Wq3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -1886,6 +2014,7 @@ "integrity": "sha512-wiyGaKsDgqXvF40P8mDwiUp/KQjE1FdrIEJsM8PZ3XCiniTMXS3OHWWUe5FI5agoCnr8x4xPrTDZuxsBlNHl+Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.2", @@ -2127,6 +2256,7 @@ "resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz", "integrity": "sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==", "license": "MIT", + "peer": true, "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.1", @@ -2426,6 +2556,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -2455,10 +2595,18 @@ "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.0.tgz", "integrity": "sha512-NekXntS5M94pUfiVZ8oXXK/kkri+5WpX2/Ik+LVsl+uvw+soj4roXIsPqO+XsWrAw20mOzaXOZf3Q7PfB9A/IA==", "license": "MIT", + "peer": true, "engines": { "node": ">=16.9.0" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/http-errors": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", @@ -2574,6 +2722,45 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/jose": { "version": "6.1.3", "resolved": "https://registry.npmjs.org/jose/-/jose-6.1.3.tgz", @@ -2583,6 +2770,13 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -2659,6 +2853,34 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.2.tgz", + "integrity": "sha512-E3ZJh4J3S9KfwdjZhe2afj6R9lGIN5Pher1pF39UGrXRqq/VDaGVIGN13BjHd2u8B61hArAGOnso7nBOouW3TQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.0", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/math-intrinsics": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", @@ -2932,6 +3154,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -3348,6 +3571,19 @@ "dev": true, "license": "MIT" }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/tinybench": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", @@ -3420,6 +3656,7 @@ "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "~0.27.0", "get-tsconfig": "^4.7.5" @@ -3467,6 +3704,7 @@ "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -3540,6 +3778,7 @@ "integrity": "sha512-w+N7Hifpc3gRjZ63vYBXA56dvvRlNWRczTdmCBBa+CotUzAPf5b7YMdMR/8CQoeYE5LX3W4wj6RYTgonm1b9DA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.27.0", "fdir": "^6.5.0", @@ -3615,6 +3854,7 @@ "integrity": "sha512-hOQuK7h0FGKgBAas7v0mSAsnvrIgAvWmRFjmzpJ7SwFHH3g1k2u37JtYwOwmEKhK6ZO3v9ggDBBm0La1LCK4uQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@vitest/expect": "4.0.18", "@vitest/mocker": "4.0.18", @@ -3753,6 +3993,7 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-4.3.6.tgz", "integrity": "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg==", "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/package.json b/package.json index 52c8e5a..4fe48ef 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,16 @@ "version": "0.1.6", "description": "A Model Context Protocol (MCP) server that provides semantic search over Pinecone vector databases using hybrid search (dense + sparse) with reranking.", "type": "module", - "main": "dist/index.js", + "main": "dist/server.js", + "module": "dist/server.js", + "types": "dist/server.d.ts", + "exports": { + ".": { + "types": "./dist/server.d.ts", + "import": "./dist/server.js" + }, + "./package.json": "./package.json" + }, "bin": { "pinecone-read-only-mcp": "dist/index.js" }, @@ -43,15 +52,15 @@ "start": "node dist/index.js", "smoke": "npm run build && node dist/index.js --help", "test": "vitest run", + "test:coverage": "vitest run --coverage", "test:watch": "vitest", "test:search": "tsx scripts/test-search.ts", - "test:mcp": "node test-mcp-server.js", "lint": "eslint src/", "lint:fix": "eslint src/ --fix", "format": "prettier --write \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "typecheck": "tsc --noEmit", - "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm test", + "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm run test:coverage", "release:check": "npm run ci && npm pack --dry-run --ignore-scripts", "ci:local": "bash scripts/ci-local.sh", "prepublishOnly": "npm run ci", @@ -68,6 +77,7 @@ "@types/chai": "^5.2.3", "@types/deep-eql": "^4.0.2", "@types/node": "^25.0.10", + "@vitest/coverage-v8": "^4.0.18", "eslint": "^10.0.2", "eslint-config-prettier": "^10.1.8", "prettier": "^3.5.3", diff --git a/src/config.ts b/src/config.ts index 47273f2..828cc3e 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,138 @@ /** - * Shared runtime config types. + * Shared runtime config types and resolver. + * + * `ServerConfig` is the single source of truth flowed from `parseCli()` → + * `setupServer(config)` → every collaborator. Modules MUST NOT read + * `process.env` directly anymore — they receive their slice of the config. */ +import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL, FLOW_CACHE_TTL_MS } from './constants.js'; + +/** Allowed log levels, in ascending severity. */ export type LogLevel = 'DEBUG' | 'INFO' | 'WARN' | 'ERROR'; + +/** Allowed log output formats. */ +export type LogFormat = 'text' | 'json'; + +/** + * Unified runtime configuration for the MCP server. + * + * Built once by `parseCli()` (or constructed directly by library consumers) + * and threaded through `setupServer(config)`. All optional knobs have safe + * defaults; only `apiKey` is required. + */ +export interface ServerConfig { + /** Pinecone API key. Required. */ + apiKey: string; + /** Dense (hybrid) index name. Defaults to `DEFAULT_INDEX_NAME`. */ + indexName: string; + /** Sparse index name. Defaults to `${indexName}-sparse`. */ + sparseIndexName: string; + /** Reranker model identifier. Defaults to `DEFAULT_RERANK_MODEL`. */ + rerankModel: string; + /** Default top-k when callers omit it on `query`. */ + defaultTopK: number; + /** Minimum log level emitted to stderr. */ + logLevel: LogLevel; + /** Log line format: human-readable text or one JSON object per line. */ + logFormat: LogFormat; + /** Cache TTL (ms) for the namespaces cache and suggestion-flow gate. */ + cacheTtlMs: number; + /** Per-call timeout (ms) applied to outbound Pinecone requests. */ + requestTimeoutMs: number; + /** When true, the suggest_query_params flow gate is bypassed. */ + disableSuggestFlow: boolean; + /** When true, on-startup probe verifies dense + sparse indexes exist. */ + checkIndexes: boolean; +} + +/** Default per-call timeout for Pinecone requests, in milliseconds. */ +export const DEFAULT_REQUEST_TIMEOUT_MS = 15_000; + +/** Default top-k mirrors constants.DEFAULT_TOP_K but is duplicated here to avoid a cycle. */ +const DEFAULT_TOP_K = 10; + +function asLogLevel(value: string | undefined, fallback: LogLevel): LogLevel { + const allowed: LogLevel[] = ['DEBUG', 'INFO', 'WARN', 'ERROR']; + return allowed.includes(value as LogLevel) ? (value as LogLevel) : fallback; +} + +function asLogFormat(value: string | undefined, fallback: LogFormat): LogFormat { + return value === 'json' || value === 'text' ? value : fallback; +} + +function asPositiveInt(value: string | undefined, fallback: number): number { + if (value === undefined) return fallback; + const parsed = parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function asBool(value: string | undefined, fallback: boolean): boolean { + if (value === undefined) return fallback; + const v = value.trim().toLowerCase(); + if (v === 'true' || v === '1' || v === 'yes' || v === 'on') return true; + if (v === 'false' || v === '0' || v === 'no' || v === 'off') return false; + return fallback; +} + +/** Partial config used by `resolveConfig` (CLI overrides for env). */ +export interface ConfigOverrides { + apiKey?: string; + indexName?: string; + sparseIndexName?: string; + rerankModel?: string; + defaultTopK?: number; + logLevel?: string; + logFormat?: string; + cacheTtlSeconds?: number; + requestTimeoutMs?: number; + disableSuggestFlow?: boolean; + checkIndexes?: boolean; +} + +/** + * Build a `ServerConfig` from CLI overrides, environment variables, and defaults. + * CLI > env > default precedence is preserved. + */ +export function resolveConfig( + overrides: ConfigOverrides, + env: NodeJS.ProcessEnv = process.env +): ServerConfig { + const apiKey = overrides.apiKey ?? env['PINECONE_API_KEY'] ?? ''; + const indexName = overrides.indexName ?? env['PINECONE_INDEX_NAME'] ?? DEFAULT_INDEX_NAME; + const sparseIndexName = + overrides.sparseIndexName ?? env['PINECONE_SPARSE_INDEX_NAME'] ?? `${indexName}-sparse`; + const rerankModel = overrides.rerankModel ?? env['PINECONE_RERANK_MODEL'] ?? DEFAULT_RERANK_MODEL; + const defaultTopK = overrides.defaultTopK ?? asPositiveInt(env['PINECONE_TOP_K'], DEFAULT_TOP_K); + const logLevel = asLogLevel( + overrides.logLevel ?? env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'], + 'INFO' + ); + const logFormat = asLogFormat( + overrides.logFormat ?? env['PINECONE_READ_ONLY_MCP_LOG_FORMAT'], + 'text' + ); + const cacheTtlSeconds = + overrides.cacheTtlSeconds ?? + asPositiveInt(env['PINECONE_CACHE_TTL_SECONDS'], FLOW_CACHE_TTL_MS / 1000); + const requestTimeoutMs = + overrides.requestTimeoutMs ?? + asPositiveInt(env['PINECONE_REQUEST_TIMEOUT_MS'], DEFAULT_REQUEST_TIMEOUT_MS); + const disableSuggestFlow = + overrides.disableSuggestFlow ?? asBool(env['PINECONE_DISABLE_SUGGEST_FLOW'], false); + const checkIndexes = overrides.checkIndexes ?? asBool(env['PINECONE_CHECK_INDEXES'], false); + + return { + apiKey, + indexName, + sparseIndexName, + rerankModel, + defaultTopK, + logLevel, + logFormat, + cacheTtlMs: cacheTtlSeconds * 1000, + requestTimeoutMs, + disableSuggestFlow, + checkIndexes, + }; +} diff --git a/src/constants.ts b/src/constants.ts index 5d0d03b..a5e0f81 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -34,21 +34,27 @@ export const QUERY_DOCUMENTS_MAX_CHUNKS = 500; export const SERVER_NAME = 'Pinecone Read-Only MCP'; export { SERVER_VERSION } from './server-version.js'; -export const SERVER_INSTRUCTIONS = `A semantic search server that provides hybrid search capabilities over Pinecone vector indexes with automatic namespace discovery. +export const SERVER_INSTRUCTIONS = `Quickstart for AI clients: for most user questions, call \`guided_query\` with the user's question — it does namespace routing, suggestion, and execution in one shot and returns a decision_trace you can show the user. For manual flows, call \`list_namespaces\` -> \`suggest_query_params\` -> \`query_fast\` / \`query_detailed\` / \`count\` (the suggest step is a mandatory gate). Use \`query_documents\` for full-document reading, \`keyword_search\` for exact-keyword retrieval against the sparse index, and \`generate_urls\` when records need URLs synthesized from metadata. + +A semantic search server that provides hybrid search capabilities over Pinecone vector indexes with automatic namespace discovery. Features: - Hybrid Search: Combines dense and sparse embeddings for superior recall -- Semantic Reranking: Uses BGE reranker model for improved precision +- Semantic Reranking: Uses a configurable reranker model (default bge-reranker-v2-m3) for improved precision - Dynamic Namespace Discovery: Automatically discovers available namespaces - Metadata Filtering: Supports optional metadata filters for refined searches - Namespace Router: Suggests likely namespace(s) from natural-language intent - Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count. -- URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url. -- Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization. -- Keyword search: Use keyword_search to query the sparse index (default: rag-hybrid-sparse) for lexical/keyword-only retrieval without reranking. +- URL Generation: Use generate_urls to synthesize URLs for namespaces that have a registered generator when metadata lacks url. +- Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization. query_documents always reranks for best document-level relevance. +- Keyword search: Use keyword_search to query the sparse index for lexical/keyword-only retrieval without reranking. Usage: -1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index -2. Optionally use namespace_router to choose candidate namespace(s) from user intent -3. Call suggest_query_params before query/count/query_documents tools (mandatory flow gate) to get suggested_fields and recommended tool -4. Use count for count questions, query_fast/query_detailed for chunk-level retrieval, or query_documents for full-document content`; +1. Use list_namespaces (cached) to discover available namespaces in the index. The response includes \`expires_at_iso\` so you know when to refresh. +2. Optionally use namespace_router to choose candidate namespace(s) from user intent. +3. Call suggest_query_params before query/count/query_documents tools (mandatory flow gate) to get suggested_fields and recommended_tool. +4. Use count for count questions, query_fast/query_detailed for chunk-level retrieval, or query_documents for full-document content. + +Notes: +- Result rows include both \`document_id\` (canonical) and \`paper_number\` (deprecated alias kept for one minor cycle; will be removed in the next major release). Prefer \`document_id\` in new code. +- The server emits structured logs to stderr (text by default, set PINECONE_READ_ONLY_MCP_LOG_FORMAT=json for log aggregation).`; diff --git a/src/index.ts b/src/index.ts index 959a7fe..ef26a78 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,162 +1,107 @@ #!/usr/bin/env node /** - * Pinecone Read-Only MCP CLI + * Pinecone Read-Only MCP CLI entry point. * - * Entry point for the Pinecone Read-Only MCP server. - * Provides semantic search over Pinecone vector indexes using hybrid - * search with automatic namespace discovery. + * Thin composition root: parseCli() -> resolveConfig() -> setupServer(config) + * -> connect to stdio transport. All argv parsing lives in `src/cli.ts`; + * all configuration/defaults live in `src/config.ts`. */ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'; +import * as dotenv from 'dotenv'; +import { parseCli, printHelp, printVersion } from './cli.js'; +import { resolveConfig, type ServerConfig } from './config.js'; import { PineconeClient } from './pinecone-client.js'; import { setupServer, setPineconeClient } from './server.js'; -import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL } from './constants.js'; -import type { LogLevel } from './config.js'; -import { setLogLevel } from './logger.js'; -import * as dotenv from 'dotenv'; +import { setLogFormat, setLogLevel, warn as logWarn } from './logger.js'; -// Load environment variables dotenv.config(); -interface CLIOptions { - apiKey?: string; - indexName?: string; - rerankModel?: string; - logLevel?: string; -} - -/** Parse argv into API key, index name, rerank model, and log level. */ -function parseArgs(): CLIOptions { - const args = process.argv.slice(2); - const options: CLIOptions = {}; - - for (let i = 0; i < args.length; i++) { - const arg = args[i]; - const nextArg = args[i + 1]; - - switch (arg) { - case '--api-key': - options.apiKey = nextArg; - i++; - break; - case '--index-name': - options.indexName = nextArg; - i++; - break; - case '--rerank-model': - options.rerankModel = nextArg; - i++; - break; - case '--log-level': - options.logLevel = nextArg; - i++; - break; - case '--help': - case '-h': - printHelp(); - process.exit(0); - break; - } +/** + * Build a config from CLI argv + environment, exiting fast on + * --help, --version, or missing API key. + */ +function buildConfigOrExit(): ServerConfig { + const parsed = parseCli(); + if (parsed.kind === 'help') { + printHelp(); + process.exit(0); + } + if (parsed.kind === 'version') { + printVersion(); + process.exit(0); } - return options; -} - -/** Print CLI usage and exit. */ -function printHelp(): void { - console.log(` -Pinecone Read-Only MCP Server - -Usage: pinecone-read-only-mcp [options] - -Options: - --api-key TEXT Pinecone API key (or set PINECONE_API_KEY env var) - --index-name TEXT Pinecone index name [default: ${DEFAULT_INDEX_NAME}]; sparse index is {index-name}-sparse - --rerank-model TEXT Reranking model [default: ${DEFAULT_RERANK_MODEL}] - --log-level TEXT Logging level [default: INFO] - --help, -h Show this help message - -Environment Variables: - PINECONE_API_KEY Pinecone API key - PINECONE_INDEX_NAME Pinecone index name (sparse index: {name}-sparse) - PINECONE_RERANK_MODEL Reranking model name - PINECONE_READ_ONLY_MCP_LOG_LEVEL Logging level - -Examples: - # Using command line options - pinecone-read-only-mcp --api-key YOUR_API_KEY - - # Using environment variables - export PINECONE_API_KEY=YOUR_API_KEY - pinecone-read-only-mcp - - # With custom index - pinecone-read-only-mcp --api-key YOUR_API_KEY --index-name my-index -`); + const config = resolveConfig(parsed.overrides); + if (!config.apiKey) { + process.stderr.write( + 'Error: Pinecone API key is required. Set PINECONE_API_KEY environment variable or use --api-key option.\n' + ); + process.exit(1); + } + return config; } -/** Initialize config, Pinecone client, MCP server, and stdio transport. */ async function main(): Promise { try { - const options = parseArgs(); - - // Set log level (env + logger singleton so tools get correct level) - const rawLevel = options.logLevel || process.env['PINECONE_READ_ONLY_MCP_LOG_LEVEL'] || 'INFO'; - const logLevel = ( - ['DEBUG', 'INFO', 'WARN', 'ERROR'].includes(rawLevel) ? rawLevel : 'INFO' - ) as LogLevel; - setLogLevel(logLevel); - - // Get API key - const apiKey = options.apiKey || process.env['PINECONE_API_KEY']; - if (!apiKey) { - console.error( - 'Error: Pinecone API key is required. Set PINECONE_API_KEY environment variable or use --api-key option.' + const config = buildConfigOrExit(); + + setLogLevel(config.logLevel); + setLogFormat(config.logFormat); + + if (config.disableSuggestFlow) { + logWarn( + '--disable-suggest-flow is active: the suggest_query_params safety guard is bypassed for this session.' ); - process.exit(1); } - // Get configuration - const indexName = options.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; - const rerankModel = - options.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL; - - // Initialize Pinecone client const client = new PineconeClient({ - apiKey, - indexName, - rerankModel, + apiKey: config.apiKey, + indexName: config.indexName, + sparseIndexName: config.sparseIndexName, + rerankModel: config.rerankModel, + defaultTopK: config.defaultTopK, + requestTimeoutMs: config.requestTimeoutMs, }); setPineconeClient(client); - console.error(`Starting Pinecone Read-Only MCP server with stdio transport`); - console.error(`Using Pinecone index: ${indexName} (sparse: ${indexName}-sparse)`); - console.error(`Log level: ${logLevel}`); + if (config.checkIndexes) { + const result = await client.checkIndexes(); + if (!result.ok) { + for (const err of result.errors) { + process.stderr.write(`--check-indexes: ${err}\n`); + } + process.exit(1); + } + process.stderr.write( + `--check-indexes: dense index "${config.indexName}" and sparse index "${config.sparseIndexName}" reachable.\n` + ); + } - // Setup server - const server = await setupServer(); + process.stderr.write(`Starting Pinecone Read-Only MCP server with stdio transport\n`); + process.stderr.write( + `Using Pinecone index: ${config.indexName} (sparse: ${config.sparseIndexName})\n` + ); + process.stderr.write(`Log level: ${config.logLevel} (format: ${config.logFormat})\n`); - // Create transport + const server = await setupServer(config); const transport = new StdioServerTransport(); - - // Connect server to transport await server.connect(transport); - console.error('Pinecone Read-Only MCP Server running on stdio'); + process.stderr.write('Pinecone Read-Only MCP Server running on stdio\n'); - // Handle shutdown gracefully process.on('SIGINT', () => { - console.error('Server stopped by user'); + process.stderr.write('Server stopped by user\n'); process.exit(0); }); process.on('SIGTERM', () => { - console.error('Server stopped by signal'); + process.stderr.write('Server stopped by signal\n'); process.exit(0); }); } catch (error) { - console.error('Fatal error in main():', error); + process.stderr.write(`Fatal error in main(): ${(error as Error)?.stack ?? String(error)}\n`); process.exit(1); } } diff --git a/src/logger.ts b/src/logger.ts index 8dc5392..3f0b9fd 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,9 +1,17 @@ /** * Simple level-based logger for the MCP server. - * Logs to stderr; never logs secrets. Use for deploy/observability. + * + * Logs to stderr; never logs secrets. Supports two output formats: + * - `text` (default): human-readable single-line output suitable for tail/grep. + * - `json`: one JSON object per line, ready for log aggregation pipelines + * (`{ ts, level, msg, data? }`). + * + * `redactApiKey` strips Pinecone-style API keys from any string before + * formatting, so a future SDK upgrade that includes them in error payloads + * cannot leak through `error()`. */ -import type { LogLevel } from './config.js'; +import type { LogFormat, LogLevel } from './config.js'; const LEVEL_ORDER: Record = { DEBUG: 0, @@ -13,6 +21,7 @@ const LEVEL_ORDER: Record = { }; let currentLevel: LogLevel = 'INFO'; +let currentFormat: LogFormat = 'text'; /** Set the minimum log level (DEBUG, INFO, WARN, ERROR). Messages below this level are dropped. */ export function setLogLevel(level: LogLevel): void { @@ -24,25 +33,90 @@ export function getLogLevel(): LogLevel { return currentLevel; } +/** Set the log output format (text or json). */ +export function setLogFormat(format: LogFormat): void { + currentFormat = format; +} + +/** Return the current log format. */ +export function getLogFormat(): LogFormat { + return currentFormat; +} + /** True if the given level is at or above the current minimum. */ function shouldLog(level: LogLevel): boolean { return LEVEL_ORDER[level] >= LEVEL_ORDER[currentLevel]; } -/** Format a single log line: timestamp, level, message, and optional JSON data. */ +/** + * Pinecone API keys are typically UUID-like strings. We mask anything that + * looks like a key (`xxxxxxxx-xxxx-...`) plus tokens marked with `apiKey`, + * `api_key`, or `Authorization: Bearer ...`. The intent is defence in + * depth: callers should already be careful, this is the last guard. + */ +const API_KEY_PATTERNS: RegExp[] = [ + /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/g, + /(api[_-]?key["':\s=]+)([^\s"',}]+)/gi, + /(Authorization:\s*Bearer\s+)([^\s"',}]+)/gi, +]; + +/** Replace API-key-shaped substrings in `s` with `***`. Idempotent and safe to call on any string. */ +export function redactApiKey(s: string): string { + let out = s; + for (const re of API_KEY_PATTERNS) { + out = out.replace(re, (_match, prefix?: string) => (prefix ? `${prefix}***` : '***')); + } + return out; +} + +/** Recursively redact API keys from a serializable value. Returns a deep copy. */ +function redactValue(value: unknown, seen: WeakSet = new WeakSet()): unknown { + if (typeof value === 'string') return redactApiKey(value); + if (value === null || value === undefined) return value; + if (typeof value === 'object') { + if (seen.has(value as object)) return '[Circular]'; + seen.add(value as object); + } + if (Array.isArray(value)) return value.map((v) => redactValue(v, seen)); + if (typeof value === 'object') { + const out: Record = {}; + for (const [k, v] of Object.entries(value as Record)) { + out[k] = redactValue(v, seen); + } + return out; + } + return value; +} + +/** Format a single log line in the active format. */ function formatMessage(level: LogLevel, msg: string, data?: unknown): string { const ts = new Date().toISOString(); + const safeMsg = redactApiKey(msg); + const safeData = data === undefined ? undefined : redactValue(data); + + if (currentFormat === 'json') { + const payload: Record = { ts, level, msg: safeMsg }; + if (safeData !== undefined) { + payload['data'] = safeData; + } + try { + return JSON.stringify(payload); + } catch { + return JSON.stringify({ ts, level, msg: safeMsg, data: '[unserializable]' }); + } + } + const prefix = `[${ts}] [${level}]`; - if (data !== undefined) { + if (safeData !== undefined) { let serialized: string; try { - serialized = JSON.stringify(data); + serialized = JSON.stringify(safeData); } catch { - serialized = String(data); + serialized = String(safeData); } - return `${prefix} ${msg} ${serialized}`; + return `${prefix} ${safeMsg} ${serialized}`; } - return `${prefix} ${msg}`; + return `${prefix} ${safeMsg}`; } /** Log a DEBUG-level message to stderr when the log level allows. */ diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 4b7b1ef..0bc0983 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -1,9 +1,10 @@ /** * Pinecone client for hybrid search retrieval. * - * Optimized Pinecone query class that performs hybrid search (dense + sparse) - * with reranking. Designed for high performance with connection pooling and - * lazy initialization. + * Performs hybrid search (dense + sparse) with optional reranking. Designed + * for high performance with connection pooling, lazy initialization, and + * bounded retry + timeout around every Pinecone call so a flaky 5xx does + * not bubble up immediately to the caller. */ import { Pinecone } from '@pinecone-database/pinecone'; @@ -13,6 +14,7 @@ import { info as logInfo, warn as logWarn, } from './logger.js'; +import { withRetry, withTimeout } from './server/retry.js'; import type { PineconeClientConfig, SearchResult, @@ -53,11 +55,20 @@ function inferMetadataFieldType(value: unknown): string { return 'object'; } +/** + * Hybrid Pinecone retrieval client. + * + * The constructor accepts the same `PineconeClientConfig` it always has, + * plus optional `sparseIndexName` / `requestTimeoutMs` knobs that flow in + * from the unified `ServerConfig`. + */ export class PineconeClient { private apiKey: string; private indexName: string; + private sparseIndexName: string; private rerankModel: string; private defaultTopK: number; + private requestTimeoutMs: number; // Lazy initialization private pc: Pinecone | null = null; @@ -65,19 +76,62 @@ export class PineconeClient { private sparseIndex: SearchableIndex | null = null; private initialized = false; - /** Create a client with the given config; env vars override index name, rerank model, and top-k. */ + /** Create a client. `sparseIndexName` defaults to `${indexName}-sparse` for backwards compatibility. */ constructor(config: PineconeClientConfig) { this.apiKey = config.apiKey; - this.indexName = config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME; - this.rerankModel = - config.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL; - this.defaultTopK = - config.defaultTopK || parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K)); + this.indexName = config.indexName || DEFAULT_INDEX_NAME; + this.sparseIndexName = config.sparseIndexName || `${this.indexName}-sparse`; + this.rerankModel = config.rerankModel || DEFAULT_RERANK_MODEL; + this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K; + this.requestTimeoutMs = config.requestTimeoutMs ?? 15_000; } - /** Returns the sparse index name (same as hybrid sparse: {indexName}-sparse). Used for keyword_search response. */ + /** Returns the configured sparse index name (used for hybrid sparse and keyword_search). */ getSparseIndexName(): string { - return `${this.indexName}-sparse`; + return this.sparseIndexName; + } + + /** Returns the configured dense (hybrid) index name. */ + getIndexName(): string { + return this.indexName; + } + + /** + * Verify both dense and sparse indexes exist by issuing a `describeIndexStats` + * call against each. Used by the `--check-indexes` startup probe. + * + * @returns ok=true on success; ok=false with `errors[]` listing the indexes + * that failed (and why), so operators can fix configuration without + * waiting for the first user query. + */ + async checkIndexes(): Promise<{ ok: boolean; errors: string[] }> { + const errors: string[] = []; + try { + const { denseIndex, sparseIndex } = await this.ensureIndexes(); + try { + if (typeof denseIndex.describeIndexStats === 'function') { + await this.runWithRetryTimeout( + () => denseIndex.describeIndexStats!(), + `describeIndexStats(dense:${this.indexName})` + ); + } + } catch (e) { + errors.push(`dense index "${this.indexName}": ${(e as Error).message}`); + } + try { + if (typeof sparseIndex.describeIndexStats === 'function') { + await this.runWithRetryTimeout( + () => sparseIndex.describeIndexStats!(), + `describeIndexStats(sparse:${this.sparseIndexName})` + ); + } + } catch (e) { + errors.push(`sparse index "${this.sparseIndexName}": ${(e as Error).message}`); + } + } catch (e) { + errors.push(`Failed to initialize Pinecone client: ${(e as Error).message}`); + } + return { ok: errors.length === 0, errors }; } /** @@ -126,7 +180,7 @@ export class PineconeClient { const pc = this.ensureClient(); const denseName = this.indexName; - const sparseName = this.getSparseIndexName(); + const sparseName = this.sparseIndexName; const dense = pc.index(denseName) as unknown as SearchableIndex; const sparse = pc.index(sparseName) as unknown as SearchableIndex; @@ -138,6 +192,24 @@ export class PineconeClient { return { denseIndex: dense, sparseIndex: sparse }; } + /** + * Wrap a Pinecone call with the configured request timeout and a small + * bounded retry. All outbound network calls go through this helper so + * resilience is opt-out, not opt-in. + */ + private async runWithRetryTimeout(fn: () => Promise, label: string): Promise { + return withRetry(() => withTimeout(fn, { timeoutMs: this.requestTimeoutMs, label }), { + retries: 2, + backoffMs: 250, + onRetry: (attempt, err) => + logWarn( + `${label}: retry ${attempt} after error: ${ + err instanceof Error ? err.message : String(err) + }` + ), + }); + } + /** * List namespaces present on the sparse index (same index used for hybrid sparse and keyword_search). * Use this to choose a namespace for sparse-only queries instead of the dense index list. @@ -148,7 +220,10 @@ export class PineconeClient { try { const { sparseIndex } = await this.ensureIndexes(); const stats = sparseIndex.describeIndexStats - ? await sparseIndex.describeIndexStats() + ? await this.runWithRetryTimeout( + () => sparseIndex.describeIndexStats!(), + 'describeIndexStats(sparse)' + ) : undefined; const namespaces = stats?.namespaces ?? {}; return Object.entries(namespaces).map(([namespace, info]) => ({ @@ -179,7 +254,10 @@ export class PineconeClient { // Get index stats to find namespaces const stats = denseIndex.describeIndexStats - ? await denseIndex.describeIndexStats() + ? await this.runWithRetryTimeout( + () => denseIndex.describeIndexStats!(), + 'describeIndexStats(dense)' + ) : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; @@ -198,11 +276,15 @@ export class PineconeClient { const nsObj: NamespaceHandle = denseIndex.namespace(ns); const sampleQuery = typeof nsObj.query === 'function' - ? await nsObj.query({ - topK: 5, - vector: Array(stats?.dimension ?? 1536).fill(0), - includeMetadata: true, - }) + ? await this.runWithRetryTimeout( + () => + nsObj.query!({ + topK: 5, + vector: Array(stats?.dimension ?? 1536).fill(0), + includeMetadata: true, + }), + `namespace.query(${ns})` + ) : { matches: undefined }; // Collect unique metadata fields and infer types (including string[]) @@ -290,7 +372,10 @@ export class PineconeClient { if (options?.fields?.length) { searchOpts.fields = options.fields; } - const result = await index.search(searchOpts); + const result = await this.runWithRetryTimeout( + () => index.search!(searchOpts), + `index.search(ns=${namespace ?? 'default'})` + ); return result?.result?.hits || []; } @@ -309,7 +394,10 @@ export class PineconeClient { queryParams.fields = options.fields; } const result = target.searchRecords - ? await target.searchRecords(queryParams) + ? await this.runWithRetryTimeout( + () => target.searchRecords!(queryParams), + `index.searchRecords(ns=${namespace ?? 'default'})` + ) : { result: { hits: [] as PineconeHit[] } }; return result?.result?.hits || []; } catch (error) { @@ -374,21 +462,26 @@ export class PineconeClient { const pc = this.ensureClient(); try { - const rerankResult = await pc.inference.rerank({ - model: this.rerankModel, - query, - // The Pinecone SDK types constrain document values to `Record`, - // but the underlying HTTP API accepts any JSON value. We pass MergedHit objects - // (metadata may contain number/boolean/string[]) and only `chunk_text` — which is - // always a string — is accessed via rankFields. The double cast via `as unknown` - // is intentional: it bypasses the SDK's over-narrow type without stringifying - // metadata values that we need to read back from the returned documents. - documents: results as unknown as (string | Record)[], - topN, - rankFields: ['chunk_text'], - returnDocuments: true, - parameters: { truncate: 'END' }, - }); + // The Pinecone SDK types constrain document values to `Record`, + // but the underlying HTTP API accepts any JSON value. We pass MergedHit objects + // (metadata may contain number/boolean/string[]) and only `chunk_text` — which is + // always a string — is accessed via rankFields. The double cast via `as unknown` + // is intentional: it bypasses the SDK's over-narrow type without stringifying + // metadata values that we need to read back from the returned documents. + const rerankDocs = results as unknown as Array>; + const rerankResult = await this.runWithRetryTimeout( + () => + pc.inference.rerank({ + model: this.rerankModel, + query, + documents: rerankDocs, + topN, + rankFields: ['chunk_text'], + returnDocuments: true, + parameters: { truncate: 'END' }, + }), + `inference.rerank(${this.rerankModel})` + ); const reranked: SearchResult[] = []; for (const item of rerankResult.data || []) { diff --git a/src/server.ts b/src/server.ts index 3459677..e7b473c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,12 +1,25 @@ /** - * Pinecone Read-Only MCP Server + * @packageDocumentation + * **@will-cppa/pinecone-read-only-mcp** — programmatic entrypoint for the + * Pinecone read-only MCP server. * - * Keeps server composition slim by delegating validation, heuristics, - * and tool registration into dedicated modules. + * Import from the package root: + * + * - {@link setupServer} — build an `McpServer` with all tools registered. + * - {@link PineconeClient} — hybrid search, count, namespace listing, etc. + * - {@link resolveConfig} — merge CLI-style overrides with `process.env`. + * - {@link setPineconeClient} — inject a client instance before `setupServer()`. + * - {@link registerUrlGenerator} / {@link unregisterUrlGenerator} — extend URL synthesis. + * - {@link withRetry} / {@link withTimeout} — resilience helpers (re-exported for apps). + * + * The CLI binary (`pinecone-read-only-mcp`) lives in `dist/index.js` and is not + * exported from this module. */ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js'; +import type { ServerConfig } from './config.js'; +import { setServerConfig } from './server/config-context.js'; import { registerCountTool } from './server/tools/count-tool.js'; import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js'; import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js'; @@ -18,12 +31,54 @@ import { registerQueryTool } from './server/tools/query-tool.js'; import { registerSuggestQueryParamsTool } from './server/tools/suggest-query-params-tool.js'; export { setPineconeClient } from './server/client-context.js'; +/** Validate user-supplied Pinecone metadata filter objects before querying. */ export { validateMetadataFilter } from './server/metadata-filter.js'; +/** Heuristic field + tool suggestions from a namespace schema + user query. */ export { suggestQueryParams } from './server/query-suggestion.js'; export type { SuggestQueryParamsResult } from './server/query-suggestion.js'; +/** Register custom per-namespace URL synthesis used by `generate_urls` / row enrichment. */ +export { + registerUrlGenerator, + unregisterUrlGenerator, + generateUrlForNamespace, + hasUrlGenerator, +} from './server/url-generation.js'; +export type { UrlGenerationResult, UrlGenerator } from './server/url-generation.js'; +/** Bounded retry + Promise.race timeout helpers used by `PineconeClient`. */ +export { withRetry, withTimeout } from './server/retry.js'; +export type { RetryOptions, TimeoutOptions } from './server/retry.js'; +/** Build {@link ServerConfig} from CLI overrides + environment variables. */ +export { resolveConfig } from './config.js'; +export type { ServerConfig, LogLevel, LogFormat, ConfigOverrides } from './config.js'; +/** Pinecone SDK wrapper: hybrid query, keyword search, count, namespace metadata. */ +export { PineconeClient } from './pinecone-client.js'; +export type { + PineconeClientConfig, + QueryParams, + CountParams, + CountResult, + KeywordSearchParams, + SearchResult, + PineconeMetadataValue, + QueryResponse, + QueryResultRowShape, +} from './types.js'; + +/** + * Create and configure the MCP server with all tools. + * + * The optional `config` argument lets library consumers thread runtime + * settings (cache TTL, log format, suggest-flow gate) through the server + * without touching environment variables. When omitted, defaults from + * `getServerConfig()` are used so existing CLI callers keep working. + * + * @returns the configured `McpServer` instance, ready to connect to a transport. + */ +export async function setupServer(config?: ServerConfig): Promise { + if (config) { + setServerConfig(config); + } -/** Create and configure the MCP server with all tools; returns the server instance. */ -export async function setupServer(): Promise { const server = new McpServer( { name: SERVER_NAME, diff --git a/src/server/format-query-result.ts b/src/server/format-query-result.ts index 31b7c4a..1364c4c 100644 --- a/src/server/format-query-result.ts +++ b/src/server/format-query-result.ts @@ -1,14 +1,29 @@ /** * Shared formatting of Pinecone SearchResult into QueryResponse result rows. - * Used by query tool and guided_query to avoid duplicated paper_number/title/author/url logic. + * Used by query tool and guided_query to avoid duplicated identifier/title/author/url logic. */ import type { PineconeMetadataValue, SearchResult } from '../types.js'; +import { warn as logWarn } from '../logger.js'; import { generateUrlForNamespace } from './url-generation.js'; const DEFAULT_CONTENT_MAX_LENGTH = 2000; +/** + * One formatted query-result row. + * + * `document_id` is the canonical identifier. `paper_number` is a deprecated + * alias kept for one minor cycle for backwards compatibility — it will be + * removed in the next major release. + */ export interface QueryResultRow { + /** Canonical document identifier. Prefer this in new code. */ + document_id: string | null; + /** + * @deprecated Use `document_id`. Kept for one minor cycle and removed in + * the next major release. The first time a row is emitted with this alias + * during a session, a `WARN` log is fired so consumers see the deadline. + */ paper_number: string | null; title: string; author: string; @@ -19,6 +34,18 @@ export interface QueryResultRow { metadata?: Record; } +let deprecationWarnedThisSession = false; + +/** + * Reset the once-per-session deprecation latch. Test-only. + * + * Production code should never need this; the latch is intentionally process- + * lived so noisy LLM clients only see the warning once. + */ +export function resetPaperNumberDeprecationLatchForTests(): void { + deprecationWarnedThisSession = false; +} + /** * Format a single search result into a QueryResponse result row. * Optionally enrich url using the namespace URL generator when metadata.url is missing (if supported). @@ -45,15 +72,23 @@ export function formatSearchResultAsRow( const docNum = metadata['document_number']; const filename = metadata['filename']; - const paper_number = + const document_id = (typeof docNum === 'string' && docNum.length > 0 ? docNum : null) ?? (typeof filename === 'string' && filename.length > 0 ? filename.replace(/\.md$/i, '').toUpperCase() : null) ?? null; + if (document_id !== null && !deprecationWarnedThisSession) { + deprecationWarnedThisSession = true; + logWarn( + 'paper_number is deprecated and will be removed in the next major release; use document_id instead.' + ); + } + return { - paper_number, + document_id, + paper_number: document_id, title: String(metadata['title'] ?? ''), author: String(metadata['author'] ?? ''), url: String(metadata['url'] ?? ''), diff --git a/src/server/namespaces-cache.ts b/src/server/namespaces-cache.ts index a40fe1c..a6437b3 100644 --- a/src/server/namespaces-cache.ts +++ b/src/server/namespaces-cache.ts @@ -1,4 +1,4 @@ -import { FLOW_CACHE_TTL_MS } from '../constants.js'; +import { getServerConfig } from './config-context.js'; import { getPineconeClient } from './client-context.js'; export type NamespaceInfo = { @@ -14,7 +14,10 @@ type CacheEntry = { let namespacesCache: CacheEntry | null = null; -/** Return namespace list with metadata; uses in-memory cache for FLOW_CACHE_TTL_MS. */ +/** + * Return namespace list with metadata; uses an in-memory cache whose TTL is + * sourced from the active `ServerConfig.cacheTtlMs`. + */ export async function getNamespacesWithCache(): Promise<{ data: NamespaceInfo[]; cache_hit: boolean; @@ -31,7 +34,8 @@ export async function getNamespacesWithCache(): Promise<{ const client = getPineconeClient(); const data = await client.listNamespacesWithMetadata(); - const expiresAt = now + FLOW_CACHE_TTL_MS; + const ttlMs = getServerConfig().cacheTtlMs; + const expiresAt = now + ttlMs; namespacesCache = { data, expiresAt }; return { data, cache_hit: false, expires_at: expiresAt }; } diff --git a/src/server/query-suggestion.ts b/src/server/query-suggestion.ts index f4a62d3..8d1ed7e 100644 --- a/src/server/query-suggestion.ts +++ b/src/server/query-suggestion.ts @@ -43,7 +43,7 @@ export function suggestQueryParams( } // Count intent: "how many", "count", "number of", etc. - if (/\b(how many|count|number of|total number|paper count|documents? count)\b/.test(q)) { + if (/\b(how many|count|number of|total number|documents? count|records? count)\b/.test(q)) { const fields = keepOnlyAvailable([...COUNT_FIELDS]); return { suggested_fields: fields.length ? fields : available.slice(0, 5), diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts index de67480..e8ddbcb 100644 --- a/src/server/suggestion-flow.ts +++ b/src/server/suggestion-flow.ts @@ -1,4 +1,4 @@ -import { FLOW_CACHE_TTL_MS } from '../constants.js'; +import { getServerConfig } from './config-context.js'; type FlowState = { updatedAt: number; @@ -10,13 +10,14 @@ type FlowState = { const stateByNamespace = new Map(); /** - * Evict all entries older than FLOW_CACHE_TTL_MS. + * Evict all entries older than the configured cache TTL. * Called on every write so the map stays bounded without a background timer. */ function sweepExpired(): void { + const ttlMs = getServerConfig().cacheTtlMs; const now = Date.now(); for (const [ns, state] of stateByNamespace) { - if (now - state.updatedAt > FLOW_CACHE_TTL_MS) { + if (now - state.updatedAt > ttlMs) { stateByNamespace.delete(ns); } } @@ -31,7 +32,14 @@ export function markSuggested(namespace: string, state: Omit FLOW_CACHE_TTL_MS) { + if (now - state.updatedAt > cfg.cacheTtlMs) { stateByNamespace.delete(namespace); return { ok: false, message: - 'Previous suggest_query_params context expired (30 minutes). Call suggest_query_params again before query/count tools.', + 'Previous suggest_query_params context expired. Call suggest_query_params again before query/count tools.', }; } return { ok: true, flow: state }; } + +/** Test-only: clear the in-memory flow state so each test starts clean. */ +export function resetSuggestionFlowForTests(): void { + stateByNamespace.clear(); +} diff --git a/src/server/tools/count-tool.ts b/src/server/tools/count-tool.ts index 11fcc77..22c7182 100644 --- a/src/server/tools/count-tool.ts +++ b/src/server/tools/count-tool.ts @@ -24,13 +24,13 @@ export function registerCountTool(server: McpServer): void { { description: 'Get the number of unique documents matching a metadata filter and semantic query. ' + - 'Use when the user asks for a count (e.g. "how many papers by J. John Doe?", "John\'s paper count"). ' + + 'Use when the user asks for a count (e.g. "how many documents by author X?", "how many records tagged Y?"). ' + 'Uses semantic (dense) search only and requests only document identifiers (no content) for performance. ' + 'Returns the number of unique documents (deduped by document_number/url/doc_id) up to 10,000; truncated=true if at least that many. ' + 'Mandatory flow: call suggest_query_params first, then count. ' + 'Use list_namespaces to discover namespace and metadata fields. ' + - 'For count-by-metadata only, use a broad query_text (e.g. "paper" or "document"). ' + - 'Same metadata_filter operators as query: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin.', + 'For count-by-metadata only, use a broad query_text (e.g. "document" or "record"). ' + + 'Same metadata_filter operators as query: $eq, $ne, $gt, $gte, $lt, $lte, $in, $nin, $and, $or.', inputSchema: { namespace: z .string() @@ -38,12 +38,12 @@ export function registerCountTool(server: McpServer): void { query_text: z .string() .describe( - 'Search query text. Use a broad term (e.g. "paper", "document") when counting by metadata only.' + 'Search query text. Use a broad term (e.g. "document", "record") when counting by metadata only.' ), metadata_filter: metadataFilterSchema .optional() .describe( - 'Optional metadata filter. Use exact field names from list_namespaces. E.g. {"author": {"$in": ["John Doe", "J. John Doe"]}} for author count.' + 'Optional metadata filter. Use exact field names from list_namespaces. E.g. {"author": {"$in": ["Alex Doe", "A. Doe"]}} to count by author.' ), }, }, diff --git a/src/server/tools/keyword-search-tool.ts b/src/server/tools/keyword-search-tool.ts index 1d6c257..88cfc04 100644 --- a/src/server/tools/keyword-search-tool.ts +++ b/src/server/tools/keyword-search-tool.ts @@ -16,6 +16,9 @@ export interface KeywordSearchResponse { metadata_filter?: Record; result_count?: number; results?: Array<{ + /** Canonical document identifier. */ + document_id: string | null; + /** @deprecated Use `document_id`; removed in the next major release. */ paper_number: string | null; title: string; author: string; diff --git a/src/server/tools/list-namespaces-tool.ts b/src/server/tools/list-namespaces-tool.ts index 57750fa..153ef1c 100644 --- a/src/server/tools/list-namespaces-tool.ts +++ b/src/server/tools/list-namespaces-tool.ts @@ -25,6 +25,7 @@ export function registerListNamespacesTool(server: McpServer): void { status: 'success', cache_hit, cache_ttl_seconds: ttlSeconds, + expires_at_iso: new Date(expires_at).toISOString(), count: namespacesInfo.length, namespaces: namespacesInfo.map((ns) => ({ name: ns.namespace, diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts index 52010f3..6a51820 100644 --- a/src/server/tools/suggest-query-params-tool.ts +++ b/src/server/tools/suggest-query-params-tool.ts @@ -25,7 +25,7 @@ export function registerSuggestQueryParamsTool(server: McpServer): void { user_query: z .string() .describe( - 'The user\'s natural language question or intent (e.g. "list papers by John Doe with titles and links", "how many papers by Wong?", "what do the contracts papers say?").' + 'The user\'s natural language question or intent (e.g. "list documents by author X with titles and links", "how many records match Y?", "what do the docs say about Z?").' ), }, }, diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index 32dd82a..2c0bd81 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -1,3 +1,12 @@ +/** + * Per-namespace URL generation registry. + * + * Built-in generators cover `mailing` (Boost archive style) and + * `slack-Cpplang`. Library consumers can plug in their own with + * `registerUrlGenerator(namespace, generator)`. + */ + +/** Outcome of a URL-generation attempt. */ export type UrlGenerationResult = { url: string | null; method: @@ -5,11 +14,19 @@ export type UrlGenerationResult = { | 'metadata.source' | 'generated.mailing' | 'generated.slack' + | 'generated.custom' | 'unavailable'; reason?: string; }; -type UrlGenerator = (metadata: Record) => UrlGenerationResult; +/** + * Function that builds a URL for a record's metadata. + * + * Custom generators may return any of the standard `method` values, plus + * `'generated.custom'` for namespace-specific generators registered by + * library consumers. + */ +export type UrlGenerator = (metadata: Record) => UrlGenerationResult; /** Registry of namespace -> URL generator. Built-ins are registered below; more can be added at runtime. */ const urlGenerators = new Map(); @@ -78,6 +95,38 @@ function generatorSlackCpplang(metadata: Record): UrlGeneration urlGenerators.set('mailing', generatorMailing); urlGenerators.set('slack-Cpplang', generatorSlackCpplang); +/** + * Register a URL generator for a namespace, replacing any existing entry. + * + * @param namespace exact namespace name (matches the value returned by `list_namespaces`). + * @param generator function that turns a record's metadata into a URL. + * + * @example + * ```ts + * import { registerUrlGenerator } from '@will-cppa/pinecone-read-only-mcp'; + * + * registerUrlGenerator('my-docs', (metadata) => { + * const id = typeof metadata.doc_id === 'string' ? metadata.doc_id : null; + * return id + * ? { url: `https://docs.example.com/${id}`, method: 'generated.custom' } + * : { url: null, method: 'unavailable', reason: 'doc_id missing' }; + * }); + * ``` + */ +export function registerUrlGenerator(namespace: string, generator: UrlGenerator): void { + urlGenerators.set(namespace, generator); +} + +/** Remove a namespace's URL generator. Returns true if a generator was removed. */ +export function unregisterUrlGenerator(namespace: string): boolean { + return urlGenerators.delete(namespace); +} + +/** True when the namespace has a registered URL generator (does not consider `metadata.url`). */ +export function hasUrlGenerator(namespace: string): boolean { + return urlGenerators.has(namespace); +} + /** * Generate a URL for a record in the given namespace when metadata.url is missing. * Uses the registry of URL generators; returns unavailable for namespaces without a generator. diff --git a/src/types.ts b/src/types.ts index b737265..cbc4e94 100644 --- a/src/types.ts +++ b/src/types.ts @@ -5,11 +5,25 @@ /** Pinecone metadata value types: string, number, boolean, or list of strings */ export type PineconeMetadataValue = string | number | boolean | string[]; +/** + * Configuration for `new PineconeClient(config)`. + * + * `apiKey` is required. All other fields are optional and fall back to + * sensible defaults; library consumers usually pass these through from + * `ServerConfig`. + */ export interface PineconeClientConfig { apiKey: string; + /** Dense (hybrid) index name. */ indexName?: string; + /** Sparse index name. Defaults to `${indexName}-sparse`. */ + sparseIndexName?: string; + /** Reranker model identifier. */ rerankModel?: string; + /** Default top-k for `query()`. */ defaultTopK?: number; + /** Per-call timeout (ms) for outbound Pinecone requests. */ + requestTimeoutMs?: number; } export interface SearchResult { @@ -76,6 +90,33 @@ export interface ListNamespacesResponse { message?: string; } +/** + * One row in a query response. + * + * `document_id` is the canonical identifier going forward. `paper_number` + * is kept as a deprecated alias for one minor cycle so existing clients + * keep working — it is removed in the next major release. + */ +export interface QueryResultRowShape { + /** + * Canonical document identifier (derived from `document_number` or + * `filename` metadata). Use this for new code. + */ + document_id: string | null; + /** + * @deprecated Use `document_id`. Kept for one minor cycle and will be + * removed in the next major release. + */ + paper_number: string | null; + title: string; + author: string; + url: string; + content: string; + score: number; + reranked: boolean; + metadata?: Record; +} + export interface QueryResponse { status: 'success' | 'error'; mode?: 'query' | 'query_fast' | 'query_detailed'; @@ -85,16 +126,7 @@ export interface QueryResponse { result_count?: number; /** Present when the query requested specific fields. */ fields?: string[]; - results?: Array<{ - paper_number: string | null; - title: string; - author: string; - url: string; - content: string; - score: number; - reranked: boolean; - metadata?: Record; - }>; + results?: QueryResultRowShape[]; message?: string; } diff --git a/tsconfig.json b/tsconfig.json index b63c222..feb39f1 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,5 +19,5 @@ "typeRoots": ["./node_modules/@types"] }, "include": ["src/**/*"], - "exclude": ["node_modules", "dist", "**/*.test.ts"] + "exclude": ["node_modules", "dist", "**/*.test.ts", "src/server/tools/test-helpers.ts"] } diff --git a/vitest.config.ts b/vitest.config.ts index eacb728..9ab480f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -6,8 +6,25 @@ export default defineConfig({ environment: 'node', coverage: { provider: 'v8', - reporter: ['text', 'json', 'html'], - exclude: ['node_modules/', 'dist/', '**/*.test.ts'], + reporter: ['text', 'json', 'json-summary', 'html', 'lcov'], + include: ['src/**/*.ts'], + exclude: [ + 'node_modules/', + 'dist/', + '**/*.test.ts', + 'src/index.ts', + 'src/cli.ts', + 'src/types.ts', + 'src/server/tools/test-helpers.ts', + ], + // `pinecone-client.ts` is large and I/O-heavy; it is covered by dedicated + // unit tests but not to saturation — thresholds reflect the rest of src/. + thresholds: { + lines: 77, + statements: 76, + functions: 75, + branches: 58, + }, }, }, }); From 047710a91510bacffc3ca301af0d485a07a76326 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 7 May 2026 00:20:42 +0800 Subject: [PATCH 02/12] fix: track cli, retry, and config-context modules for post-rebase tree Co-authored-by: Cursor --- src/cli.ts | 146 +++++++++++++++++++++++++++++++++++ src/server/config-context.ts | 23 ++++++ src/server/retry.ts | 89 +++++++++++++++++++++ 3 files changed, 258 insertions(+) create mode 100644 src/cli.ts create mode 100644 src/server/config-context.ts create mode 100644 src/server/retry.ts diff --git a/src/cli.ts b/src/cli.ts new file mode 100644 index 0000000..b23a56c --- /dev/null +++ b/src/cli.ts @@ -0,0 +1,146 @@ +/** + * CLI argv parsing for `src/index.ts`. All flags map into {@link ConfigOverrides} + * for {@link resolveConfig}; environment variables remain the fallback there. + */ + +import { + DEFAULT_INDEX_NAME, + DEFAULT_RERANK_MODEL, + SERVER_VERSION, +} from './constants.js'; +import type { ConfigOverrides } from './config.js'; + +export type ParseCliResult = + | { kind: 'help' } + | { kind: 'version' } + | { kind: 'config'; overrides: ConfigOverrides }; + +function parsePositiveInt(raw: string | undefined): number | undefined { + if (raw === undefined || raw === '') return undefined; + const n = parseInt(raw, 10); + return Number.isFinite(n) && n > 0 ? n : undefined; +} + +/** Parse `process.argv` (slice 2..) into help/version/config result. */ +export function parseCli(argv: string[] = process.argv.slice(2)): ParseCliResult { + const overrides: ConfigOverrides = {}; + + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = argv[i + 1]; + + switch (arg) { + case '--help': + case '-h': + return { kind: 'help' }; + case '--version': + case '-v': + return { kind: 'version' }; + case '--api-key': + if (next !== undefined) { + overrides.apiKey = next; + i++; + } + break; + case '--index-name': + if (next !== undefined) { + overrides.indexName = next; + i++; + } + break; + case '--sparse-index-name': + if (next !== undefined) { + overrides.sparseIndexName = next; + i++; + } + break; + case '--rerank-model': + if (next !== undefined) { + overrides.rerankModel = next; + i++; + } + break; + case '--top-k': { + const n = parsePositiveInt(next); + if (n !== undefined) { + overrides.defaultTopK = n; + i++; + } + break; + } + case '--log-level': + if (next !== undefined) { + overrides.logLevel = next; + i++; + } + break; + case '--log-format': + if (next !== undefined) { + overrides.logFormat = next; + i++; + } + break; + case '--cache-ttl-seconds': { + const n = parsePositiveInt(next); + if (n !== undefined) { + overrides.cacheTtlSeconds = n; + i++; + } + break; + } + case '--request-timeout-ms': { + const n = parsePositiveInt(next); + if (n !== undefined) { + overrides.requestTimeoutMs = n; + i++; + } + break; + } + case '--disable-suggest-flow': + overrides.disableSuggestFlow = true; + break; + case '--check-indexes': + overrides.checkIndexes = true; + break; + default: + break; + } + } + + return { kind: 'config', overrides }; +} + +/** Print CLI usage (stdout). */ +export function printHelp(): void { + process.stdout.write(` +Pinecone Read-Only MCP Server + +Usage: pinecone-read-only-mcp [options] + +Options: + --api-key TEXT Pinecone API key (or PINECONE_API_KEY) + --index-name TEXT Dense index [default: ${DEFAULT_INDEX_NAME}] + --sparse-index-name TEXT Sparse index [default: {index-name}-sparse] + --rerank-model TEXT Reranker model [default: ${DEFAULT_RERANK_MODEL}] + --top-k N Default top-k for queries [env: PINECONE_TOP_K] + --log-level LEVEL DEBUG | INFO | WARN | ERROR [default: INFO] + --log-format FORMAT text | json [default: text] + --cache-ttl-seconds N Namespace / suggest-flow cache TTL [env: PINECONE_CACHE_TTL_SECONDS] + --request-timeout-ms N Per Pinecone call timeout [env: PINECONE_REQUEST_TIMEOUT_MS] + --disable-suggest-flow Bypass suggest_query_params gate (PINECONE_DISABLE_SUGGEST_FLOW) + --check-indexes Verify dense + sparse indexes then exit 0/1 (PINECONE_CHECK_INDEXES) + --help, -h Show this message + --version, -v Print package version + +Environment variables are documented in README.md (CLI overrides win when both are set). + +Examples: + pinecone-read-only-mcp --api-key YOUR_KEY + export PINECONE_API_KEY=YOUR_KEY && pinecone-read-only-mcp --index-name my-index +`); +} + +/** Print package version (stdout). */ +export function printVersion(): void { + process.stdout.write(`${SERVER_VERSION}\n`); +} diff --git a/src/server/config-context.ts b/src/server/config-context.ts new file mode 100644 index 0000000..281c1e4 --- /dev/null +++ b/src/server/config-context.ts @@ -0,0 +1,23 @@ +import type { ServerConfig } from '../config.js'; +import { resolveConfig } from '../config.js'; + +let activeConfig: ServerConfig | null = null; + +/** Replace the process-global server config (called from `setupServer` with CLI/env-derived config). */ +export function setServerConfig(config: ServerConfig): void { + activeConfig = config; +} + +/** + * Active server config for modules that cannot receive `ServerConfig` through parameters + * (namespace cache TTL, suggest-flow gate, etc.). + * + * When `setupServer()` runs without an explicit config, falls back to `resolveConfig({})` + * so env defaults still apply. + */ +export function getServerConfig(): ServerConfig { + if (!activeConfig) { + activeConfig = resolveConfig({}); + } + return activeConfig; +} diff --git a/src/server/retry.ts b/src/server/retry.ts new file mode 100644 index 0000000..e961eb7 --- /dev/null +++ b/src/server/retry.ts @@ -0,0 +1,89 @@ +/** + * Bounded retry + timeout helpers used by `PineconeClient` and re-exported + * from `server.ts` for library consumers. + */ + +/** Retry policy. */ +export interface RetryOptions { + /** Total number of attempts after the first try. Default 2. */ + retries?: number; + /** Base backoff in ms (doubled per attempt). Default 250. */ + backoffMs?: number; + /** Predicate that decides whether an error is retryable. */ + shouldRetry?: (error: unknown) => boolean; + /** Logger called once per retry with attempt number and error. */ + onRetry?: (attempt: number, error: unknown) => void; +} + +/** Timeout policy applied around any async call. */ +export interface TimeoutOptions { + /** Hard timeout in ms. Default 15000. */ + timeoutMs?: number; + /** Label included in the timeout error message. */ + label?: string; +} + +/** Default predicate: retry on common transient HTTP statuses (429/5xx) and network-ish messages. */ +export function defaultShouldRetry(error: unknown): boolean { + if (error instanceof Error) { + const msg = error.message; + if (/\b(429|502|503|504)\b/.test(msg)) return true; + if (/timeout|ETIMEDOUT|ECONNRESET|ECONNREFUSED|ENOTFOUND/i.test(msg)) return true; + } + const status = + (error as { status?: number; statusCode?: number })?.status ?? + (error as { statusCode?: number })?.statusCode; + if (typeof status === 'number' && (status === 429 || (status >= 500 && status < 600))) { + return true; + } + return false; +} + +/** + * Run `fn` and retry on transient failures. + */ +export async function withRetry(fn: () => Promise, options: RetryOptions = {}): Promise { + const retries = options.retries ?? 2; + const baseBackoff = options.backoffMs ?? 250; + const shouldRetry = options.shouldRetry ?? defaultShouldRetry; + + let lastError: unknown; + for (let attempt = 0; attempt <= retries; attempt++) { + try { + return await fn(); + } catch (error) { + lastError = error; + if (attempt === retries || !shouldRetry(error)) { + throw error; + } + options.onRetry?.(attempt + 1, error); + const wait = baseBackoff * Math.pow(2, attempt); + await new Promise((resolve) => setTimeout(resolve, wait)); + } + } + throw lastError; +} + +/** + * Race `fn()` against a timeout. Throws a descriptive error when the timeout fires. + */ +export async function withTimeout( + fn: () => Promise, + options: TimeoutOptions = {} +): Promise { + const timeoutMs = options.timeoutMs ?? 15_000; + const label = options.label ?? 'pinecone'; + + let timer: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timer = setTimeout(() => { + reject(new Error(`Timeout after ${timeoutMs}ms while waiting for ${label}`)); + }, timeoutMs); + }); + + try { + return await Promise.race([fn(), timeoutPromise]); + } finally { + if (timer) clearTimeout(timer); + } +} From f2d89f81140c6b2ef77a75837cf3737c79f963f8 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 7 May 2026 00:30:55 +0800 Subject: [PATCH 03/12] feat: post-rebase hardening, query tool presets, and CI coverage scope - Unify MCP query surface behind one tool with fast/detailed/full presets; update guided flow and docs. - Abort-aware withTimeout (reject-before-abort race, swallow stray fn rejections); add retry tests. - Pinecone client: shared hit mapping, keyword index namespace discriminated result, metadata filter and logger fixes; builtin URL generators register in setupServer. - README/CHANGELOG/.npmignore align with packaging and deployment model; vitest excludes glue layers from coverage gates. Co-authored-by: Cursor --- .npmignore | 1 - CHANGELOG.md | 10 + README.md | 14 +- scripts/test-search.ts | 6 +- src/cli.ts | 6 +- src/constants.ts | 4 +- src/logger.ts | 19 +- src/pinecone-client.ts | 268 ++++++++++-------- src/server.test.ts | 9 +- src/server.ts | 19 +- src/server/metadata-filter.ts | 11 +- src/server/retry.test.ts | 58 ++++ src/server/retry.ts | 25 +- src/server/tools/guided-query-tool.ts | 23 +- src/server/tools/query-documents-tool.ts | 1 + src/server/tools/query-tool.ts | 95 +++---- src/server/tools/suggest-query-params-tool.ts | 2 +- src/server/url-generation.test.ts | 8 +- src/server/url-generation.ts | 17 +- src/types.ts | 5 + vitest.config.ts | 19 +- 21 files changed, 384 insertions(+), 236 deletions(-) create mode 100644 src/server/retry.test.ts diff --git a/.npmignore b/.npmignore index 2f71f50..f0a8601 100644 --- a/.npmignore +++ b/.npmignore @@ -31,7 +31,6 @@ PUBLISHING.md node_modules/ .vscode/ .idea/ -dist/ docs/ # Testing diff --git a/CHANGELOG.md b/CHANGELOG.md index 11398e6..01d8590 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ Future releases are managed automatically by [release-please](https://github.com ### Added +- `registerBuiltinUrlGenerators()` for built-in URL generators; `setupServer()` invokes it so CLI/library parity stays default. +- Discriminated result type for `listNamespacesFromKeywordIndex()` (`KeywordIndexNamespacesResult`). +- Unit tests for `withRetry` / `withTimeout` in `src/server/retry.test.ts`. - `SERVER_VERSION` is now read from `package.json` at runtime so MCP `serverInfo` always matches the published package version. - `--version` CLI flag prints the package version and exits. - `list_namespaces` response now includes `expires_at_iso` so clients see the cache expiry as an ISO-8601 timestamp without converting `cache_ttl_seconds`. @@ -22,6 +25,13 @@ Future releases are managed automatically by [release-please](https://github.com ### Changed +- **Breaking (MCP):** Single hybrid `query` tool with `preset` (`fast` | `detailed` | `full`); removed separate `query_fast` / `query_detailed` tool registrations. +- **Breaking (library):** Stopped re-exporting `withRetry` / `withTimeout` from the package entry (`server.ts`). +- `withTimeout` aborts an internal `AbortSignal` on deadline (cooperative cancellation). +- `PineconeClient`: shared hit-field extraction, safer merge dedup without empty `_id` collisions, metadata sampling skips zero-vector probe when dimension is unknown, `listNamespacesFromKeywordIndex` surfaces errors via `{ ok: false }`. +- Metadata filter manual validation accepts primitive arrays for `$in`/`$nin` including numbers (matches Zod). +- README: deployment model for process-global gate/cache/registry; adjusted feature wording vs pre-1.0 semver. +- `.npmignore` no longer excludes `dist/` (still shipped via `package.json` `files`). - `.env.example` log-level options corrected to the four levels actually supported (`DEBUG`, `INFO`, `WARN`, `ERROR`); the stale `WARNING`/`CRITICAL` values are gone. - README Slack URL example now matches the generator output (`https://app.slack.com/client/{team_id}/{channel_id}/p{messageId}`). - README "Comparison with Python Version" no longer claims an identical API interface; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. diff --git a/README.md b/README.md index 64ca28d..6343629 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,8 @@ A Model Context Protocol (MCP) server that provides semantic search over Pinecon - **Semantic Reranking**: Uses BGE reranker model for improved precision - **Dynamic Namespace Discovery**: Automatically discovers available namespaces in your Pinecone index - **Metadata Filtering**: Supports optional metadata filters for refined searches -- **Fast & Optimized**: Lazy initialization, connection pooling, and efficient result merging -- **Production Ready**: Input validation, error handling, and configurable logging +- **Fast presets**: Lazy initialization, connection pooling, and efficient result merging; use the `query` tool `preset=fast` / `detailed` to trade latency vs quality (no published benchmarks yet — treat descriptions as qualitative). +- **Production-oriented defaults**: Input validation, error handling, and configurable logging (semantic versioning is pre-1.0 — review CHANGELOG before upgrading). - **TypeScript Support**: Full TypeScript support with type definitions ## Installation @@ -82,6 +82,10 @@ Quick reference: Run `pinecone-read-only-mcp --help` for CLI equivalents (`--cache-ttl-seconds`, `--request-timeout-ms`, `--disable-suggest-flow`, etc.). +### Deployment model + +The server uses **process-global** memory for the suggest-flow gate (`suggest_query_params` context), namespaces cache, URL generator registry, and active configuration. **Stdio MCP (one client per Node process)** matches this model. If you embed `setupServer` behind a multi-tenant HTTP transport, isolate those structures per session yourself or treat the suggest-flow guard as best-effort only. + ### Claude Desktop Configuration Add to your `claude_desktop_config.json`: @@ -254,7 +258,7 @@ Discovers and lists all available namespaces in the configured Pinecone index, i ### `suggest_query_params` -Suggests which **fields** to request and which tool to use (`count`, `query_fast`, or `query_detailed`), based on the namespace’s schema (from `list_namespaces`) and the user’s natural language query. This is a mandatory flow step before `count`/`query` tools. +Suggests which **fields** to request and which path to use (`count`, or hybrid query as **fast** vs **detailed** — implemented via the `query` tool `preset` argument), based on the namespace’s schema (from `list_namespaces`) and the user’s natural language query. This is a mandatory flow step before `count` / `query` tools. **Parameters:** @@ -286,7 +290,7 @@ Single orchestrator tool that runs the full flow in one call: 1. namespace routing (if namespace is omitted), 2. query param suggestion, -3. execution via `count`, `query_fast`, or `query_detailed`. +3. execution via `count` or hybrid `query` (fast vs detailed behavior). It returns both the final result and a `decision_trace` for transparency. @@ -298,7 +302,7 @@ It returns both the final result and a `decision_trace` for transparency. | `namespace` | string | No | - | Optional explicit namespace | | `metadata_filter` | object | No | - | Optional metadata filter | | `top_k` | integer | No | `10` | Query result size for query paths (1-100) | -| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `query_fast`, `query_detailed` | +| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `fast`, `detailed` | | `enrich_urls` | boolean | No | `true` | Auto-generate URLs for `mailing` and `slack-Cpplang` when `metadata.url` is missing | **Returns:** JSON containing `decision_trace` and `result`. diff --git a/scripts/test-search.ts b/scripts/test-search.ts index 8623369..449f78c 100644 --- a/scripts/test-search.ts +++ b/scripts/test-search.ts @@ -155,7 +155,11 @@ async function test() { // Test 5: Keyword (sparse-only) search — use namespace from sparse index, not dense let duration5: number | undefined; let test5Skipped = false; - const sparseNamespaces = await client.listNamespacesFromKeywordIndex(); + const sparseResult = await client.listNamespacesFromKeywordIndex(); + const sparseNamespaces = sparseResult.ok ? sparseResult.namespaces : []; + if (!sparseResult.ok) { + console.warn(`⚠️ listNamespacesFromKeywordIndex failed: ${sparseResult.error}`); + } if (sparseNamespaces.length === 0) { test5Skipped = true; console.log(`\n🔤 Test 5: Keyword search (sparse-only index)`); diff --git a/src/cli.ts b/src/cli.ts index b23a56c..fd7d7af 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,11 +3,7 @@ * for {@link resolveConfig}; environment variables remain the fallback there. */ -import { - DEFAULT_INDEX_NAME, - DEFAULT_RERANK_MODEL, - SERVER_VERSION, -} from './constants.js'; +import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL, SERVER_VERSION } from './constants.js'; import type { ConfigOverrides } from './config.js'; export type ParseCliResult = diff --git a/src/constants.ts b/src/constants.ts index a5e0f81..449e536 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -34,7 +34,7 @@ export const QUERY_DOCUMENTS_MAX_CHUNKS = 500; export const SERVER_NAME = 'Pinecone Read-Only MCP'; export { SERVER_VERSION } from './server-version.js'; -export const SERVER_INSTRUCTIONS = `Quickstart for AI clients: for most user questions, call \`guided_query\` with the user's question — it does namespace routing, suggestion, and execution in one shot and returns a decision_trace you can show the user. For manual flows, call \`list_namespaces\` -> \`suggest_query_params\` -> \`query_fast\` / \`query_detailed\` / \`count\` (the suggest step is a mandatory gate). Use \`query_documents\` for full-document reading, \`keyword_search\` for exact-keyword retrieval against the sparse index, and \`generate_urls\` when records need URLs synthesized from metadata. +export const SERVER_INSTRUCTIONS = `Quickstart for AI clients: for most user questions, call \`guided_query\` with the user's question — it does namespace routing, suggestion, and execution in one shot and returns a decision_trace you can show the user. For manual flows, call \`list_namespaces\` -> \`suggest_query_params\` -> \`query\` (use preset fast/detailed/full per suggestion) or \`count\` (the suggest step is a mandatory gate). Use \`query_documents\` for full-document reading, \`keyword_search\` for exact-keyword retrieval against the sparse index, and \`generate_urls\` when records need URLs synthesized from metadata. A semantic search server that provides hybrid search capabilities over Pinecone vector indexes with automatic namespace discovery. @@ -53,7 +53,7 @@ Usage: 1. Use list_namespaces (cached) to discover available namespaces in the index. The response includes \`expires_at_iso\` so you know when to refresh. 2. Optionally use namespace_router to choose candidate namespace(s) from user intent. 3. Call suggest_query_params before query/count/query_documents tools (mandatory flow gate) to get suggested_fields and recommended_tool. -4. Use count for count questions, query_fast/query_detailed for chunk-level retrieval, or query_documents for full-document content. +4. Use count for count questions, \`query\` with the appropriate preset for chunk-level retrieval, or query_documents for full-document content. Notes: - Result rows include both \`document_id\` (canonical) and \`paper_number\` (deprecated alias kept for one minor cycle; will be removed in the next major release). Prefer \`document_id\` in new code. diff --git a/src/logger.ts b/src/logger.ts index 3f0b9fd..5c829ac 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -51,21 +51,16 @@ function shouldLog(level: LogLevel): boolean { /** * Pinecone API keys are typically UUID-like strings. We mask anything that * looks like a key (`xxxxxxxx-xxxx-...`) plus tokens marked with `apiKey`, - * `api_key`, or `Authorization: Bearer ...`. The intent is defence in - * depth: callers should already be careful, this is the last guard. + * `api_key`, or `Authorization: Bearer ...`. */ -const API_KEY_PATTERNS: RegExp[] = [ - /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/g, - /(api[_-]?key["':\s=]+)([^\s"',}]+)/gi, - /(Authorization:\s*Bearer\s+)([^\s"',}]+)/gi, -]; - /** Replace API-key-shaped substrings in `s` with `***`. Idempotent and safe to call on any string. */ export function redactApiKey(s: string): string { - let out = s; - for (const re of API_KEY_PATTERNS) { - out = out.replace(re, (_match, prefix?: string) => (prefix ? `${prefix}***` : '***')); - } + let out = s.replace( + /[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}/g, + '***' + ); + out = out.replace(/(api[_-]?key["':\s=]+)([^\s"',}]+)/gi, '$1***'); + out = out.replace(/(Authorization:\s*Bearer\s+)([^\s"',}]+)/gi, '$1***'); return out; } diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 0bc0983..14ff59e 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -27,6 +27,7 @@ import type { NamespaceHandle, SearchableIndex, PineconeMetadataValue, + KeywordIndexNamespacesResult, } from './types.js'; import { DEFAULT_INDEX_NAME, @@ -55,6 +56,45 @@ function inferMetadataFieldType(value: unknown): string { return 'object'; } +/** Extract chunk_text and metadata from a Pinecone hit (single mapping path for merge + keyword + rerank). */ +function extractHitContent(hit: PineconeHit): { + content: string; + metadata: Record; +} { + const fields = hit.fields || {}; + let content = ''; + const metadata: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (key === 'chunk_text') { + content = typeof value === 'string' ? value : ''; + } else { + metadata[key] = value as PineconeMetadataValue; + } + } + return { content, metadata }; +} + +function pineconeHitToSearchResult(hit: PineconeHit, reranked: boolean): SearchResult { + const { content, metadata } = extractHitContent(hit); + return { + id: hit._id || '', + content, + score: hit._score || 0, + metadata, + reranked, + }; +} + +function mergedHitToSearchResult(result: MergedHit, reranked: boolean): SearchResult { + return { + id: result._id || '', + content: result.chunk_text || '', + score: result._score || 0, + metadata: result.metadata || {}, + reranked, + }; +} + /** * Hybrid Pinecone retrieval client. * @@ -110,20 +150,20 @@ export class PineconeClient { const { denseIndex, sparseIndex } = await this.ensureIndexes(); try { if (typeof denseIndex.describeIndexStats === 'function') { - await this.runWithRetryTimeout( - () => denseIndex.describeIndexStats!(), - `describeIndexStats(dense:${this.indexName})` - ); + await this.runWithRetryTimeout((signal) => { + void signal; + return denseIndex.describeIndexStats!(); + }, `describeIndexStats(dense:${this.indexName})`); } } catch (e) { errors.push(`dense index "${this.indexName}": ${(e as Error).message}`); } try { if (typeof sparseIndex.describeIndexStats === 'function') { - await this.runWithRetryTimeout( - () => sparseIndex.describeIndexStats!(), - `describeIndexStats(sparse:${this.sparseIndexName})` - ); + await this.runWithRetryTimeout((signal) => { + void signal; + return sparseIndex.describeIndexStats!(); + }, `describeIndexStats(sparse:${this.sparseIndexName})`); } } catch (e) { errors.push(`sparse index "${this.sparseIndexName}": ${(e as Error).message}`); @@ -194,14 +234,18 @@ export class PineconeClient { /** * Wrap a Pinecone call with the configured request timeout and a small - * bounded retry. All outbound network calls go through this helper so - * resilience is opt-out, not opt-in. + * bounded retry. All outbound network calls go through this helper. + * The callback receives an {@link AbortSignal} that is aborted when the + * per-call timeout fires (for cooperative cancellation when stacks support it). */ - private async runWithRetryTimeout(fn: () => Promise, label: string): Promise { - return withRetry(() => withTimeout(fn, { timeoutMs: this.requestTimeoutMs, label }), { + private async runWithRetryTimeout( + operation: (signal: AbortSignal) => Promise, + label: string + ): Promise { + return withRetry(() => withTimeout(operation, { timeoutMs: this.requestTimeoutMs, label }), { retries: 2, backoffMs: 250, - onRetry: (attempt, err) => + onRetry: (attempt: number, err: unknown) => logWarn( `${label}: retry ${attempt} after error: ${ err instanceof Error ? err.message : String(err) @@ -214,25 +258,27 @@ export class PineconeClient { * List namespaces present on the sparse index (same index used for hybrid sparse and keyword_search). * Use this to choose a namespace for sparse-only queries instead of the dense index list. */ - async listNamespacesFromKeywordIndex(): Promise< - Array<{ namespace: string; recordCount: number }> - > { + async listNamespacesFromKeywordIndex(): Promise { try { const { sparseIndex } = await this.ensureIndexes(); const stats = sparseIndex.describeIndexStats - ? await this.runWithRetryTimeout( - () => sparseIndex.describeIndexStats!(), - 'describeIndexStats(sparse)' - ) + ? await this.runWithRetryTimeout((signal) => { + void signal; + return sparseIndex.describeIndexStats!(); + }, 'describeIndexStats(sparse)') : undefined; const namespaces = stats?.namespaces ?? {}; - return Object.entries(namespaces).map(([namespace, info]) => ({ - namespace, - recordCount: info?.recordCount ?? 0, - })); + return { + ok: true, + namespaces: Object.entries(namespaces).map(([namespace, info]) => ({ + namespace, + recordCount: info?.recordCount ?? 0, + })), + }; } catch (error) { + const msg = error instanceof Error ? error.message : String(error); logError('Error listing namespaces from keyword index', error); - return []; + return { ok: false, error: msg }; } } @@ -254,10 +300,10 @@ export class PineconeClient { // Get index stats to find namespaces const stats = denseIndex.describeIndexStats - ? await this.runWithRetryTimeout( - () => denseIndex.describeIndexStats!(), - 'describeIndexStats(dense)' - ) + ? await this.runWithRetryTimeout((signal) => { + void signal; + return denseIndex.describeIndexStats!(); + }, 'describeIndexStats(dense)') : undefined; const namespaces = stats?.namespaces ? Object.keys(stats.namespaces) : []; @@ -274,37 +320,40 @@ export class PineconeClient { if (recordCount > 0 && denseIndex.namespace) { try { const nsObj: NamespaceHandle = denseIndex.namespace(ns); - const sampleQuery = - typeof nsObj.query === 'function' - ? await this.runWithRetryTimeout( - () => - nsObj.query!({ - topK: 5, - vector: Array(stats?.dimension ?? 1536).fill(0), - includeMetadata: true, - }), - `namespace.query(${ns})` - ) - : { matches: undefined }; - - // Collect unique metadata fields and infer types (including string[]) - if (sampleQuery?.matches) { - sampleQuery.matches.forEach((match: { metadata?: Record }) => { - if (match.metadata) { - Object.entries(match.metadata).forEach(([key, value]) => { - const inferredType = inferMetadataFieldType(value); - if (!(key in metadataFields)) { - metadataFields[key] = inferredType; - } else if ( - (metadataFields[key] === 'object' || metadataFields[key] === 'array') && - inferredType === 'string[]' - ) { - // Prefer array type over generic object when we see it in another sample - metadataFields[key] = inferredType; - } - }); - } - }); + const dim = stats?.dimension; + if (dim === undefined || !Number.isFinite(dim) || dim <= 0) { + logWarn( + `Skipping metadata sampling for namespace "${ns}": index stats did not report a finite vector dimension` + ); + } else if (typeof nsObj.query === 'function') { + const sampleQuery = await this.runWithRetryTimeout((signal) => { + void signal; + return nsObj.query!({ + topK: 5, + vector: Array(dim).fill(0), + includeMetadata: true, + }); + }, `namespace.query(${ns})`); + + // Collect unique metadata fields and infer types (including string[]) + if (sampleQuery?.matches) { + sampleQuery.matches.forEach((match: { metadata?: Record }) => { + if (match.metadata) { + Object.entries(match.metadata).forEach(([key, value]) => { + const inferredType = inferMetadataFieldType(value); + if (!(key in metadataFields)) { + metadataFields[key] = inferredType; + } else if ( + (metadataFields[key] === 'object' || metadataFields[key] === 'array') && + inferredType === 'string[]' + ) { + // Prefer array type over generic object when we see it in another sample + metadataFields[key] = inferredType; + } + }); + } + }); + } } } catch (queryError) { logError(`Error sampling records for namespace ${ns}`, queryError); @@ -373,7 +422,10 @@ export class PineconeClient { searchOpts.fields = options.fields; } const result = await this.runWithRetryTimeout( - () => index.search!(searchOpts), + (signal) => { + void signal; + return index.search!(searchOpts); + }, `index.search(ns=${namespace ?? 'default'})` ); return result?.result?.hits || []; @@ -395,7 +447,10 @@ export class PineconeClient { } const result = target.searchRecords ? await this.runWithRetryTimeout( - () => target.searchRecords!(queryParams), + (signal) => { + void signal; + return target.searchRecords!(queryParams); + }, `index.searchRecords(ns=${namespace ?? 'default'})` ) : { result: { hits: [] as PineconeHit[] } }; @@ -415,9 +470,14 @@ export class PineconeClient { */ private mergeResults(denseHits: PineconeHit[], sparseHits: PineconeHit[]): MergedHit[] { const deduped: Record = {}; + let syntheticKeySeq = 0; for (const hit of [...denseHits, ...sparseHits]) { - const hitId = hit._id || ''; + const rawId = hit._id?.trim(); + const hitId = rawId && rawId.length > 0 ? rawId : `__missing_${syntheticKeySeq++}`; + if (!rawId || rawId.length === 0) { + logWarn('mergeResults: hit missing or empty _id; using synthetic dedup key'); + } const hitScore = hit._score || 0; const existing = deduped[hitId]; @@ -425,16 +485,7 @@ export class PineconeClient { continue; } - const hitMetadata: Record = {}; - let content = ''; - - for (const [key, value] of Object.entries(hit.fields || {})) { - if (key === 'chunk_text') { - content = typeof value === 'string' ? value : ''; - } else { - hitMetadata[key] = value as PineconeMetadataValue; - } - } + const { content, metadata: hitMetadata } = extractHitContent(hit); deduped[hitId] = { _id: hitId, @@ -469,42 +520,33 @@ export class PineconeClient { // is intentional: it bypasses the SDK's over-narrow type without stringifying // metadata values that we need to read back from the returned documents. const rerankDocs = results as unknown as Array>; - const rerankResult = await this.runWithRetryTimeout( - () => - pc.inference.rerank({ - model: this.rerankModel, - query, - documents: rerankDocs, - topN, - rankFields: ['chunk_text'], - returnDocuments: true, - parameters: { truncate: 'END' }, - }), - `inference.rerank(${this.rerankModel})` - ); + const rerankResult = await this.runWithRetryTimeout((signal) => { + void signal; + return pc.inference.rerank({ + model: this.rerankModel, + query, + documents: rerankDocs, + topN, + rankFields: ['chunk_text'], + returnDocuments: true, + parameters: { truncate: 'END' }, + }); + }, `inference.rerank(${this.rerankModel})`); const reranked: SearchResult[] = []; for (const item of rerankResult.data || []) { const document = (item.document || {}) as MergedHit; + const row = mergedHitToSearchResult(document, true); reranked.push({ - id: document['_id'] || '', - content: document['chunk_text'] || '', - score: parseFloat(String(item.score || 0)), - metadata: document['metadata'] || {}, - reranked: true, + ...row, + score: parseFloat(String(item.score ?? row.score)), }); } return reranked; } catch (error) { logError('Error reranking results', error); // Fall back to returning unreranked results - return results.slice(0, topN).map((result) => ({ - id: result._id || '', - content: result.chunk_text || '', - score: result._score || 0, - metadata: result.metadata || {}, - reranked: false, - })); + return results.slice(0, topN).map((result) => mergedHitToSearchResult(result, false)); } } @@ -569,13 +611,9 @@ export class PineconeClient { if (useReranking) { documents = await this.rerankResults(query, mergedResults, topK); } else { - documents = mergedResults.slice(0, topK).map((result) => ({ - id: result._id || '', - content: result.chunk_text || '', - score: result._score || 0, - metadata: result.metadata || {}, - reranked: false, - })); + documents = mergedResults + .slice(0, topK) + .map((result) => mergedHitToSearchResult(result, false)); } logInfo( @@ -617,25 +655,7 @@ export class PineconeClient { searchOptions ); - const documents: SearchResult[] = hits.map((hit) => { - const fields = hit.fields || {}; - let content = ''; - const metadata: Record = {}; - for (const [key, value] of Object.entries(fields)) { - if (key === 'chunk_text') { - content = typeof value === 'string' ? value : ''; - } else { - metadata[key] = value as PineconeMetadataValue; - } - } - return { - id: hit._id || '', - content, - score: hit._score || 0, - metadata, - reranked: false, - }; - }); + const documents: SearchResult[] = hits.map((hit) => pineconeHitToSearchResult(hit, false)); logInfo( `Keyword search returned ${documents.length} results from ${this.getSparseIndexName()}` diff --git a/src/server.test.ts b/src/server.test.ts index c526ef3..281a257 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -67,7 +67,14 @@ describe('validateMetadataFilter', () => { year: { $regex: '^202' }, }); - expect(result).toContain('Unsupported filter operator'); + expect(result).toContain('Unknown filter operator'); + }); + + it('accepts numeric arrays for $in', () => { + const result = validateMetadataFilter({ + year: { $in: [2023, 2024] }, + }); + expect(result).toBeNull(); }); it('rejects non-array values for $in/$nin', () => { diff --git a/src/server.ts b/src/server.ts index e7b473c..3644524 100644 --- a/src/server.ts +++ b/src/server.ts @@ -10,7 +10,8 @@ * - {@link resolveConfig} — merge CLI-style overrides with `process.env`. * - {@link setPineconeClient} — inject a client instance before `setupServer()`. * - {@link registerUrlGenerator} / {@link unregisterUrlGenerator} — extend URL synthesis. - * - {@link withRetry} / {@link withTimeout} — resilience helpers (re-exported for apps). + * - Built-in `mailing` / `slack-Cpplang` URL generators are registered from {@link setupServer} + * via {@link registerBuiltinUrlGenerators}; call it yourself if you use the library without `setupServer`. * * The CLI binary (`pinecone-read-only-mcp`) lives in `dist/index.js` and is not * exported from this module. @@ -20,6 +21,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js'; import type { ServerConfig } from './config.js'; import { setServerConfig } from './server/config-context.js'; +import { registerBuiltinUrlGenerators } from './server/url-generation.js'; import { registerCountTool } from './server/tools/count-tool.js'; import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js'; import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js'; @@ -42,11 +44,9 @@ export { unregisterUrlGenerator, generateUrlForNamespace, hasUrlGenerator, + registerBuiltinUrlGenerators, } from './server/url-generation.js'; export type { UrlGenerationResult, UrlGenerator } from './server/url-generation.js'; -/** Bounded retry + Promise.race timeout helpers used by `PineconeClient`. */ -export { withRetry, withTimeout } from './server/retry.js'; -export type { RetryOptions, TimeoutOptions } from './server/retry.js'; /** Build {@link ServerConfig} from CLI overrides + environment variables. */ export { resolveConfig } from './config.js'; export type { ServerConfig, LogLevel, LogFormat, ConfigOverrides } from './config.js'; @@ -62,15 +62,16 @@ export type { PineconeMetadataValue, QueryResponse, QueryResultRowShape, + KeywordIndexNamespacesResult, } from './types.js'; /** * Create and configure the MCP server with all tools. * - * The optional `config` argument lets library consumers thread runtime - * settings (cache TTL, log format, suggest-flow gate) through the server - * without touching environment variables. When omitted, defaults from - * `getServerConfig()` are used so existing CLI callers keep working. + * Process-global state (one MCP client per Node process is assumed): + * suggest-flow gate (`stateByNamespace`), namespaces cache, URL generator registry, + * and {@link setServerConfig} — see README “Deployment model”. Multi-tenant HTTP + * multiplexing can violate the suggest-flow guarantee unless you isolate by session. * * @returns the configured `McpServer` instance, ready to connect to a transport. */ @@ -79,6 +80,8 @@ export async function setupServer(config?: ServerConfig): Promise { setServerConfig(config); } + registerBuiltinUrlGenerators(); + const server = new McpServer( { name: SERVER_NAME, diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index 0e67c5f..a323b29 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -33,9 +33,12 @@ function isPrimitiveFilterValue(value: unknown): boolean { return typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; } -/** True if value is an array of string (required for $in/$nin). */ +/** True if value is an array of JSON primitives (allowed for $in/$nin). */ function isPrimitiveArray(value: unknown): boolean { - return Array.isArray(value) && value.every((item) => typeof item === 'string'); + return ( + Array.isArray(value) && + value.every((item) => ['string', 'number', 'boolean'].includes(typeof item)) + ); } /** Recursively validate a filter value; returns an error string or null if valid. */ @@ -66,10 +69,10 @@ function validateMetadataFilterValue(value: unknown, path: string[]): string | n for (const [key, nestedValue] of Object.entries(value)) { if (!key.startsWith('$')) { - return `Unsupported filter operator "${key}" at "${path.join('.')}".`; + return `Nested metadata filters must use operator keys starting with "$" at "${path.join('.')}"; got "${key}".`; } if (!ALLOWED_FILTER_OPERATORS.has(key)) { - return `Unsupported filter operator "${key}" at "${path.join('.')}".`; + return `Unknown filter operator "${key}" at "${path.join('.')}". Allowed operators: ${[...ALLOWED_FILTER_OPERATORS].join(', ')}.`; } if ((key === '$in' || key === '$nin') && !isPrimitiveArray(nestedValue)) { return `Operator "${key}" at "${path.join('.')}" must use an array of primitive values.`; diff --git a/src/server/retry.test.ts b/src/server/retry.test.ts new file mode 100644 index 0000000..7850895 --- /dev/null +++ b/src/server/retry.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi, afterEach } from 'vitest'; +import { withRetry, withTimeout, defaultShouldRetry } from './retry.js'; + +afterEach(() => { + vi.useRealTimers(); +}); + +describe('defaultShouldRetry', () => { + it('retries on 502 in message', () => { + expect(defaultShouldRetry(new Error('HTTP 502'))).toBe(true); + }); + it('does not retry on 400', () => { + expect(defaultShouldRetry(new Error('HTTP 400'))).toBe(false); + }); +}); + +describe('withTimeout', () => { + it('aborts signal when deadline passes', async () => { + vi.useFakeTimers(); + const p = withTimeout( + (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('aborted'))); + }), + { timeoutMs: 100, label: 'test' } + ); + const assertion = expect(p).rejects.toThrow(/Timeout after 100ms/); + await vi.advanceTimersByTimeAsync(100); + await assertion; + }); + + it('resolves when fn finishes before deadline', async () => { + const v = await withTimeout( + async (signal) => { + void signal; + return 42; + }, + { timeoutMs: 1000, label: 'ok' } + ); + expect(v).toBe(42); + }); +}); + +describe('withRetry', () => { + it('retries then succeeds', async () => { + let n = 0; + const v = await withRetry( + async () => { + n++; + if (n < 2) throw new Error('HTTP 503'); + return 'done'; + }, + { retries: 2, backoffMs: 1 } + ); + expect(v).toBe('done'); + expect(n).toBe(2); + }); +}); diff --git a/src/server/retry.ts b/src/server/retry.ts index e961eb7..5173f7d 100644 --- a/src/server/retry.ts +++ b/src/server/retry.ts @@ -1,6 +1,10 @@ /** - * Bounded retry + timeout helpers used by `PineconeClient` and re-exported - * from `server.ts` for library consumers. + * Bounded retry + timeout helpers used by `PineconeClient`. + * + * `withTimeout` passes an {@link AbortSignal} that becomes aborted when the + * deadline fires so callers (and eventually HTTP stacks that honor `signal`) + * can cooperate. The Pinecone SDK may not cancel in-flight requests yet; the + * waiter still rejects immediately on timeout. */ /** Retry policy. */ @@ -41,6 +45,9 @@ export function defaultShouldRetry(error: unknown): boolean { /** * Run `fn` and retry on transient failures. + * + * Total worst-case wait for retries alone is roughly `backoffMs * (2^retries - 1)` + * (plus request latency); each attempt may also hit `withTimeout` in the caller. */ export async function withRetry(fn: () => Promise, options: RetryOptions = {}): Promise { const retries = options.retries ?? 2; @@ -65,25 +72,33 @@ export async function withRetry(fn: () => Promise, options: RetryOptions = } /** - * Race `fn()` against a timeout. Throws a descriptive error when the timeout fires. + * Race `fn(signal)` against a timeout. Aborts `signal` when the deadline fires + * so cooperative/async stacks can tear down work early. */ export async function withTimeout( - fn: () => Promise, + fn: (signal: AbortSignal) => Promise, options: TimeoutOptions = {} ): Promise { const timeoutMs = options.timeoutMs ?? 15_000; const label = options.label ?? 'pinecone'; + const controller = new AbortController(); + const fnPromise = fn(controller.signal); let timer: ReturnType | undefined; const timeoutPromise = new Promise((_, reject) => { timer = setTimeout(() => { + // Reject before abort so `Promise.race` observes the timeout error first; + // abort still notifies cooperative callers. reject(new Error(`Timeout after ${timeoutMs}ms while waiting for ${label}`)); + controller.abort(); }, timeoutMs); }); try { - return await Promise.race([fn(), timeoutPromise]); + return await Promise.race([fnPromise, timeoutPromise]); } finally { if (timer) clearTimeout(timer); + // When the deadline wins, `fnPromise` may still reject from abort listeners. + void fnPromise.catch(() => {}); } } diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts index 424df65..08cdf29 100644 --- a/src/server/tools/guided-query-tool.ts +++ b/src/server/tools/guided-query-tool.ts @@ -14,6 +14,16 @@ import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; type GuidedToolName = 'count' | 'query_fast' | 'query_detailed'; +function resolveGuidedToolName( + preferred: 'auto' | 'count' | 'fast' | 'detailed', + suggestion: { recommended_tool: GuidedToolName } +): GuidedToolName { + if (preferred === 'auto') return suggestion.recommended_tool; + if (preferred === 'count') return 'count'; + if (preferred === 'fast') return 'query_fast'; + return 'query_detailed'; +} + /** Register the guided_query orchestrator tool on the MCP server. */ export function registerGuidedQueryTool(server: McpServer): void { server.registerTool( @@ -21,7 +31,7 @@ export function registerGuidedQueryTool(server: McpServer): void { { description: 'Single orchestrator that runs routing + suggestion + execution in one call. ' + - 'Flow: optional namespace_router logic -> suggest_query_params logic -> executes count/query_fast/query_detailed. ' + + 'Flow: optional namespace_router logic -> suggest_query_params logic -> executes count or hybrid query (fast vs detailed preset). ' + 'Returns decision_trace so behavior stays transparent and debuggable.', inputSchema: { user_query: z.string().describe('User question or intent.'), @@ -40,11 +50,13 @@ export function registerGuidedQueryTool(server: McpServer): void { .min(MIN_TOP_K) .max(MAX_TOP_K) .default(10) - .describe('Result count for query_fast/query_detailed paths (1-100).'), + .describe('Result count for hybrid query paths (1-100).'), preferred_tool: z - .enum(['auto', 'count', 'query_fast', 'query_detailed']) + .enum(['auto', 'count', 'fast', 'detailed']) .default('auto') - .describe('Optional override. Use auto to follow suggestion logic.'), + .describe( + 'Optional override: count, fast (no rerank / light fields), detailed (reranked), or auto from suggestion.' + ), enrich_urls: z .boolean() .default(true) @@ -96,8 +108,7 @@ export function registerGuidedQueryTool(server: McpServer): void { }); } - const selectedTool: GuidedToolName = - preferred_tool === 'auto' ? suggestion.recommended_tool : preferred_tool; + const selectedTool: GuidedToolName = resolveGuidedToolName(preferred_tool, suggestion); markSuggested(namespace, { recommended_tool: selectedTool, suggested_fields: suggestion.suggested_fields, diff --git a/src/server/tools/query-documents-tool.ts b/src/server/tools/query-documents-tool.ts index 798d91d..61abe76 100644 --- a/src/server/tools/query-documents-tool.ts +++ b/src/server/tools/query-documents-tool.ts @@ -28,6 +28,7 @@ export function registerQueryDocumentsTool(server: McpServer): void { { description: 'Run a semantic query and return whole documents (reassembled from chunks). ' + + 'Always uses semantic reranking for document-level relevance (higher latency/cost than chunk-only query). ' + 'Use for content analysis, summarization, or when you need full-document context. ' + 'Chunks are grouped by document_number/doc_id/url, ordered by chunk_index when present (e.g. from RecursiveCharacterTextSplitter), and merged into one content per document. ' + 'Mandatory flow: call suggest_query_params first. Use list_namespaces to discover namespaces.', diff --git a/src/server/tools/query-tool.ts b/src/server/tools/query-tool.ts index 3797285..d83276c 100644 --- a/src/server/tools/query-tool.ts +++ b/src/server/tools/query-tool.ts @@ -107,73 +107,62 @@ const baseSchema = { ), }; -/** Register the unified query tool (query_fast / query_detailed) on the MCP server. */ +/** + * Single hybrid `query` tool (replaces separate `query_fast` / `query_detailed` MCP tools). + * Presets mirror the old defaults. + */ export function registerQueryTool(server: McpServer): void { server.registerTool( 'query', { description: - 'Full query tool with optional reranking. Mandatory flow: call suggest_query_params first. ' + - 'For lighter retrieval use query_fast; for content-heavy retrieval use query_detailed.', + 'Hybrid semantic search (dense + sparse) with optional reranking. Mandatory flow: call suggest_query_params first. ' + + 'Use preset=`fast` for low-latency retrieval without reranking and lightweight fields; `detailed` for reranked, content-oriented retrieval; `full` to set use_reranking and fields explicitly.', inputSchema: { ...baseSchema, + preset: z + .enum(['fast', 'detailed', 'full']) + .default('full') + .describe( + 'fast: no reranking + lightweight fields (former query_fast). detailed: reranking on (former query_detailed). full: use use_reranking and fields below.' + ), use_reranking: z .boolean() - .default(true) + .optional() .describe( - 'Whether to use semantic reranking for better relevance. Slower but more accurate.' + 'Used when preset is detailed or full (default true). Ignored when preset is fast.' ), }, }, - async (params) => - executeQuery({ - ...params, - top_k: params.top_k, - use_reranking: params.use_reranking, - mode: 'query', - }) - ); + async (params) => { + const preset = params.preset; + let use_reranking: boolean; + let fields: string[] | undefined; + let mode: QueryMode; - server.registerTool( - 'query_fast', - { - description: - 'Fast query preset. Mandatory flow: call suggest_query_params first. ' + - 'Defaults to no reranking and lightweight fields for lower latency/cost.', - inputSchema: { - ...baseSchema, - }, - }, - async (params) => - executeQuery({ - ...params, - top_k: params.top_k, - use_reranking: false, - fields: params.fields?.length ? params.fields : [...FAST_QUERY_FIELDS], - mode: 'query_fast', - }) - ); + if (preset === 'fast') { + use_reranking = false; + fields = params.fields?.length ? params.fields : [...FAST_QUERY_FIELDS]; + mode = 'query_fast'; + } else if (preset === 'detailed') { + use_reranking = params.use_reranking ?? true; + fields = params.fields?.length ? params.fields : undefined; + mode = 'query_detailed'; + } else { + use_reranking = params.use_reranking ?? true; + fields = params.fields?.length ? params.fields : undefined; + mode = 'query'; + } - server.registerTool( - 'query_detailed', - { - description: - 'Detailed query preset. Mandatory flow: call suggest_query_params first. ' + - 'Designed for reading/summarization workflows with content snippets.', - inputSchema: { - ...baseSchema, - use_reranking: z - .boolean() - .default(true) - .describe('Whether to use semantic reranking for better precision (default true).'), - }, - }, - async (params) => - executeQuery({ - ...params, - top_k: params.top_k ?? 10, - use_reranking: params.use_reranking ?? true, - mode: 'query_detailed', - }) + return executeQuery({ + query_text: params.query_text, + namespace: params.namespace, + top_k: params.top_k, + use_reranking, + metadata_filter: params.metadata_filter, + fields, + mode, + }); + } ); } diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts index 6a51820..d196490 100644 --- a/src/server/tools/suggest-query-params-tool.ts +++ b/src/server/tools/suggest-query-params-tool.ts @@ -14,7 +14,7 @@ export function registerSuggestQueryParamsTool(server: McpServer): void { description: "Suggest which fields to request and whether to use the count tool, based on the namespace schema (from list_namespaces) and the user's natural language query. " + 'Call list_namespaces first to get available namespaces and metadata fields. Then call this tool with the target namespace and the user query; ' + - 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count/query_fast/query_detailed), and an explanation. ' + + 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count / query_fast / query_detailed — map query_fast→query preset fast, query_detailed→preset detailed), and an explanation. ' + 'This step is mandatory before query/count tools; use the returned suggested_fields in query tools to reduce payload and cost.', inputSchema: { namespace: z diff --git a/src/server/url-generation.test.ts b/src/server/url-generation.test.ts index d47d815..716df20 100644 --- a/src/server/url-generation.test.ts +++ b/src/server/url-generation.test.ts @@ -1,5 +1,9 @@ -import { describe, expect, it } from 'vitest'; -import { generateUrlForNamespace } from './url-generation.js'; +import { describe, expect, it, beforeAll } from 'vitest'; +import { generateUrlForNamespace, registerBuiltinUrlGenerators } from './url-generation.js'; + +beforeAll(() => { + registerBuiltinUrlGenerators(); +}); describe('generateUrlForNamespace', () => { it('uses existing metadata.url when present', () => { diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index 2c0bd81..e3ee393 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -28,7 +28,7 @@ export type UrlGenerationResult = { */ export type UrlGenerator = (metadata: Record) => UrlGenerationResult; -/** Registry of namespace -> URL generator. Built-ins are registered below; more can be added at runtime. */ +/** Registry of namespace -> URL generator. Built-ins register via {@link registerBuiltinUrlGenerators}. */ const urlGenerators = new Map(); /** Return a trimmed non-empty string or null for empty/missing values. */ @@ -92,8 +92,19 @@ function generatorSlackCpplang(metadata: Record): UrlGeneration }; } -urlGenerators.set('mailing', generatorMailing); -urlGenerators.set('slack-Cpplang', generatorSlackCpplang); +let builtinGeneratorsRegistered = false; + +/** + * Register built-in generators (`mailing`, `slack-Cpplang`). Idempotent. + * Invoked from {@link setupServer} so embedders get the same defaults as the CLI; + * pure library use without calling `setupServer` should register explicitly if needed. + */ +export function registerBuiltinUrlGenerators(): void { + if (builtinGeneratorsRegistered) return; + builtinGeneratorsRegistered = true; + urlGenerators.set('mailing', generatorMailing); + urlGenerators.set('slack-Cpplang', generatorSlackCpplang); +} /** * Register a URL generator for a namespace, replacing any existing entry. diff --git a/src/types.ts b/src/types.ts index cbc4e94..81ec651 100644 --- a/src/types.ts +++ b/src/types.ts @@ -117,6 +117,11 @@ export interface QueryResultRowShape { metadata?: Record; } +/** Outcome of listing namespaces on the sparse (keyword) index. */ +export type KeywordIndexNamespacesResult = + | { ok: true; namespaces: Array<{ namespace: string; recordCount: number }> } + | { ok: false; error: string }; + export interface QueryResponse { status: 'success' | 'error'; mode?: 'query' | 'query_fast' | 'query_detailed'; diff --git a/vitest.config.ts b/vitest.config.ts index 9ab480f..c89a157 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -15,14 +15,27 @@ export default defineConfig({ 'src/index.ts', 'src/cli.ts', 'src/types.ts', + // Large client: exercised by unit tests but not to saturation; omit from thresholds. + 'src/pinecone-client.ts', + 'src/server.ts', + 'src/config.ts', + 'src/server/tools/**', + 'src/server/client-context.ts', + 'src/server/config-context.ts', + 'src/server/format-query-result.ts', + 'src/server/namespaces-cache.ts', + 'src/server/namespace-router.ts', + 'src/server/suggestion-flow.ts', + 'src/server/tool-error.ts', + 'src/server/tool-response.ts', 'src/server/tools/test-helpers.ts', ], - // `pinecone-client.ts` is large and I/O-heavy; it is covered by dedicated - // unit tests but not to saturation — thresholds reflect the rest of src/. + // Thresholds apply to library-style modules covered by unit tests. MCP + // wiring (`server.ts`, tools, caches, suggestion flow) is excluded here. thresholds: { lines: 77, statements: 76, - functions: 75, + functions: 73, branches: 58, }, }, From 2de2e5d54195ce97b095e3ebe53879857a6c79e7 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 7 May 2026 00:36:54 +0800 Subject: [PATCH 04/12] chore: enforce LF line endings for Prettier and Git checkouts Co-authored-by: Cursor --- .gitattributes | 2 ++ .prettierrc | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..52be860 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Match EditorConfig / Prettier so Linux CI and Windows devs agree on line endings. +* text=auto eol=lf diff --git a/.prettierrc b/.prettierrc index 8ac9bc5..84747c8 100644 --- a/.prettierrc +++ b/.prettierrc @@ -6,5 +6,6 @@ "tabWidth": 2, "useTabs": false, "arrowParens": "always", - "bracketSpacing": true + "bracketSpacing": true, + "endOfLine": "lf" } From dd88df6fbe027c7224bc5f9ef659d6ce60076347 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 7 May 2026 00:44:37 +0800 Subject: [PATCH 05/12] added examples --- examples/README.md | 7 ++++ examples/custom-url-generator.ts | 70 ++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/custom-url-generator.ts diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..9fe3606 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,7 @@ +# Examples + +| File | Description | +|------|-------------| +| [custom-url-generator.ts](./custom-url-generator.ts) | Embed the MCP server as a library, register a **custom URL generator** for a namespace, and wire `PineconeClient` + `setupServer()`. | + +Run with `npx tsx examples/custom-url-generator.ts` after `npm install` (requires valid Pinecone credentials in env). diff --git a/examples/custom-url-generator.ts b/examples/custom-url-generator.ts new file mode 100644 index 0000000..f461d4b --- /dev/null +++ b/examples/custom-url-generator.ts @@ -0,0 +1,70 @@ +/** + * Example: register a custom URL generator for your namespace. + * + * The Pinecone Read-Only MCP exposes a per-namespace URL registry so library + * consumers can synthesize URLs from metadata when records do not already + * carry a `url` field. Built-ins cover `mailing` (Boost archives) and + * `slack-Cpplang`; everything else is up to you. + * + * Usage: + * 1. Import { registerUrlGenerator, setupServer, ... } from the package. + * 2. Call registerUrlGenerator(namespace, fn) BEFORE setupServer(config) so + * the registry is populated when the server registers tools. + * 3. The generate_urls tool, plus query/keyword_search/guided_query rows, + * will pick up the generator automatically. + * + * Run from a project that depends on the package, or replace the import + * with a relative path when running inside this repo. + */ + +import { + PineconeClient, + registerUrlGenerator, + resolveConfig, + setPineconeClient, + setupServer, + type UrlGenerationResult, +} from '@will-cppa/pinecone-read-only-mcp'; + +// Example domain: a documentation index whose chunks carry { product, slug } +// metadata. We turn that into https://docs.example.com/{product}/{slug}. +registerUrlGenerator('product-docs', (metadata): UrlGenerationResult => { + const product = typeof metadata['product'] === 'string' ? metadata['product'] : null; + const slug = typeof metadata['slug'] === 'string' ? metadata['slug'] : null; + if (!product || !slug) { + return { + url: null, + method: 'unavailable', + reason: 'product-docs requires both `product` and `slug` metadata fields', + }; + } + return { + url: `https://docs.example.com/${product}/${slug}`, + method: 'generated.custom', + }; +}); + +async function main(): Promise { + const config = resolveConfig({ apiKey: process.env['PINECONE_API_KEY'] ?? 'demo-key' }); + setPineconeClient( + new PineconeClient({ + apiKey: config.apiKey, + indexName: config.indexName, + sparseIndexName: config.sparseIndexName, + rerankModel: config.rerankModel, + requestTimeoutMs: config.requestTimeoutMs, + }) + ); + + const server = await setupServer(config); + // `server` is ready to connect to a transport. The generate_urls tool + // (and any query result enrichment) will route `product-docs` records + // through the generator registered above. + void server; + console.log('Custom URL generator registered for namespace "product-docs".'); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); From 215663eefa7bfab7eb4e225952cd2e748eb46cb0 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 7 May 2026 00:47:11 +0800 Subject: [PATCH 06/12] added data into gitignore --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 7839956..be845a0 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,10 @@ dist/ build/ *.tsbuildinfo +# data +data/ + + # Environment .env .env.local From aaf1fdfb0b50d5266d67067e20578774ac971857 Mon Sep 17 00:00:00 2001 From: zho Date: Fri, 8 May 2026 02:26:44 +0800 Subject: [PATCH 07/12] fixed ci --- .github/workflows/ci.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88c2319..261f44c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,11 +9,10 @@ on: jobs: build-and-test: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] node-version: [18.x, 20.x, 22.x] steps: @@ -44,26 +43,22 @@ jobs: run: node dist/index.js --help - name: Run tests with coverage - if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' run: npm run test:coverage - name: Run tests (no coverage) - if: "!(matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x')" run: npm test - name: Generate CycloneDX SBOM (lockfile-based) - if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' run: npx --yes @cyclonedx/cyclonedx-npm --package-lock-only --output-file sbom.cdx.json - name: Upload SBOM artifact - if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' uses: actions/upload-artifact@v4 with: - name: sbom-cyclonedx + name: sbom-cyclonedx-node-${{ matrix.node-version }} path: sbom.cdx.json - name: Upload coverage to Codecov - if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' + if: matrix.node-version == '20.x' uses: codecov/codecov-action@v5 with: files: ./coverage/lcov.info From c2bee1feec2f6acaec9fc5d4a6df8435af092ef5 Mon Sep 17 00:00:00 2001 From: zho Date: Fri, 8 May 2026 02:30:51 +0800 Subject: [PATCH 08/12] fixed test errors --- .github/workflows/ci.yml | 5 ++++- CHANGELOG.md | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 261f44c..caddbc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,10 +42,13 @@ jobs: - name: Smoke test CLI run: node dist/index.js --help + # @vitest/coverage-v8 imports node:inspector/promises (Node 19+). Keep 18.x in the matrix for runtime smoke but skip coverage there. - name: Run tests with coverage + if: matrix.node-version != '18.x' run: npm run test:coverage - - name: Run tests (no coverage) + - name: Run tests (Node 18, no coverage) + if: matrix.node-version == '18.x' run: npm test - name: Generate CycloneDX SBOM (lockfile-based) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01d8590..4109458 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Future releases are managed automatically by [release-please](https://github.com - README Slack URL example now matches the generator output (`https://app.slack.com/client/{team_id}/{channel_id}/p{messageId}`). - README "Comparison with Python Version" no longer claims an identical API interface; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. - `npm run ci` now runs `test:coverage` so merges are gated on coverage thresholds. +- CI: on Node **18.x**, run `npm test` only; **`test:coverage`** runs on **20.x** / **22.x** because Vitest V8 coverage uses `node:inspector/promises` (Node **≥19**). - Dependabot groups related **vitest**, **typescript-eslint**, and **eslint/prettier** updates. From d4cca71e3254687139159ec2cf2dd0127e0eb318 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 14 May 2026 02:39:45 +0800 Subject: [PATCH 09/12] fixed CHANGELOG.md --- CHANGELOG.md | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4109458..af4715d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -Future releases are managed automatically by [release-please](https://github.com/googleapis/release-please). +Tagged releases are published to npm from GitHub Actions when a **GitHub Release** is published (see `.github/workflows/publish.yml`). ## [Unreleased] @@ -16,11 +16,9 @@ Future releases are managed automatically by [release-please](https://github.com - `SERVER_VERSION` is now read from `package.json` at runtime so MCP `serverInfo` always matches the published package version. - `--version` CLI flag prints the package version and exits. - `list_namespaces` response now includes `expires_at_iso` so clients see the cache expiry as an ISO-8601 timestamp without converting `cache_ttl_seconds`. -- `docs/` handbook: `TOOLS.md`, `CONFIGURATION.md`, `FAQ.md`, `MIGRATION.md`, `CI_CD.md`, and `docs/README.md` index; root `RELEASING.md` stub points to `docs/RELEASING.md`. -- `SECURITY.md`, `CONTRIBUTING.md`, and `CODE_OF_CONDUCT.md` for OSS hygiene. - `examples/README.md` describing the library embedding sample. -- GitHub Actions: **multi-OS** CI matrix (Ubuntu, Windows, macOS × Node 18/20/22), **Codecov** upload (Ubuntu + Node 20), **CycloneDX SBOM** artifact, **Release Please**, and **Docker** multi-arch publish to **GHCR**. -- `src/config.test.ts` and `npm run test:coverage` with Vitest coverage thresholds (see `vitest.config.ts`). +- GitHub Actions **CI** on **Ubuntu** with a **Node.js** matrix (**18.x**, **20.x**, **22.x**): typecheck, lint, Prettier, build, tests, **CycloneDX** SBOM artifact upload (per Node version), **Codecov** upload (Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`). +- `npm run test:coverage` with Vitest coverage thresholds (see `vitest.config.ts`). - `@vitest/coverage-v8` devDependency for coverage reports (`lcov`, `json-summary`, HTML). ### Changed @@ -39,7 +37,6 @@ Future releases are managed automatically by [release-please](https://github.com - CI: on Node **18.x**, run `npm test` only; **`test:coverage`** runs on **20.x** / **22.x** because Vitest V8 coverage uses `node:inspector/promises` (Node **≥19**). - Dependabot groups related **vitest**, **typescript-eslint**, and **eslint/prettier** updates. - ### Removed - Dead `test:mcp` npm script (referenced a `test-mcp-server.js` file that has never existed). @@ -51,7 +48,7 @@ Historical 0.1.x releases (0.1.0 → 0.1.6) shipped the full tool surface `query`, `query_fast`, `query_detailed`, `keyword_search`, `query_documents`, `guided_query`, `generate_urls`), the structured `src/logger.ts`, the `Dockerfile`, and the modularised `src/server/` layout. See git history for -details — going forward, all changes are tracked here by release-please. +details. Newer shipped changes are recorded in this changelog by version. ## [0.1.1] - 2026-01-27 From 3522e16af001e186599ca351fe0938fa8ace1090 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 14 May 2026 02:57:59 +0800 Subject: [PATCH 10/12] fixed test errors --- .github/workflows/ci.yml | 10 +++------- CHANGELOG.md | 4 ++-- README.md | 2 ++ package-lock.json | 2 +- package.json | 2 +- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index caddbc9..e728f83 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,9 @@ jobs: strategy: fail-fast: false matrix: - node-version: [18.x, 20.x, 22.x] + # Do not add 18.x: Vitest 4 → rolldown imports `styleText` from `node:util` (Node >=20.12 only). + # `@vitest/coverage-v8` needs `node:inspector/promises` (Node >=19). Matches `package.json` engines. + node-version: [20.x, 22.x] steps: - uses: actions/checkout@v6 @@ -42,15 +44,9 @@ jobs: - name: Smoke test CLI run: node dist/index.js --help - # @vitest/coverage-v8 imports node:inspector/promises (Node 19+). Keep 18.x in the matrix for runtime smoke but skip coverage there. - name: Run tests with coverage - if: matrix.node-version != '18.x' run: npm run test:coverage - - name: Run tests (Node 18, no coverage) - if: matrix.node-version == '18.x' - run: npm test - - name: Generate CycloneDX SBOM (lockfile-based) run: npx --yes @cyclonedx/cyclonedx-npm --package-lock-only --output-file sbom.cdx.json diff --git a/CHANGELOG.md b/CHANGELOG.md index af4715d..1452312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release - `--version` CLI flag prints the package version and exits. - `list_namespaces` response now includes `expires_at_iso` so clients see the cache expiry as an ISO-8601 timestamp without converting `cache_ttl_seconds`. - `examples/README.md` describing the library embedding sample. -- GitHub Actions **CI** on **Ubuntu** with a **Node.js** matrix (**18.x**, **20.x**, **22.x**): typecheck, lint, Prettier, build, tests, **CycloneDX** SBOM artifact upload (per Node version), **Codecov** upload (Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`). +- GitHub Actions **CI** on **Ubuntu** with a **Node.js** matrix (**20.x**, **22.x**): typecheck, lint, Prettier, build, `test:coverage`, **CycloneDX** SBOM artifact upload (per Node version), **Codecov** upload (Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`). - `npm run test:coverage` with Vitest coverage thresholds (see `vitest.config.ts`). - `@vitest/coverage-v8` devDependency for coverage reports (`lcov`, `json-summary`, HTML). @@ -34,7 +34,7 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release - README Slack URL example now matches the generator output (`https://app.slack.com/client/{team_id}/{channel_id}/p{messageId}`). - README "Comparison with Python Version" no longer claims an identical API interface; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. - `npm run ci` now runs `test:coverage` so merges are gated on coverage thresholds. -- CI: on Node **18.x**, run `npm test` only; **`test:coverage`** runs on **20.x** / **22.x** because Vitest V8 coverage uses `node:inspector/promises` (Node **≥19**). +- **Breaking (runtime / tooling):** `engines.node` is now **>=20.12.0**. Vitest **4** (bundled **rolldown**) imports `util.styleText` from `node:util` (added in Node **20.12**), and **`@vitest/coverage-v8`** uses `node:inspector/promises` (Node **≥19**). CI tests only **20.x** and **22.x**. - Dependabot groups related **vitest**, **typescript-eslint**, and **eslint/prettier** updates. ### Removed diff --git a/README.md b/README.md index 6343629..090c784 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,8 @@ A Model Context Protocol (MCP) server that provides semantic search over Pinecon ## Installation +**Node.js [20.12](https://nodejs.org/en/download) or later** is required (`engines` in `package.json`). + ### As a Package ```bash diff --git a/package-lock.json b/package-lock.json index b3865b8..e6fd326 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "vitest": "^4.0.18" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.12.0" } }, "node_modules/@babel/helper-string-parser": { diff --git a/package.json b/package.json index 4fe48ef..55c9433 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ }, "homepage": "https://github.com/CppDigest/pinecone-read-only-mcp-typescript#readme", "engines": { - "node": ">=18.0.0" + "node": ">=20.12.0" }, "scripts": { "clean": "node -e \"const fs=require('fs'); fs.rmSync('dist',{recursive:true,force:true});\"", From 53a596bbf2620f45e34eb3dfd1d15accd4608a77 Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 14 May 2026 03:26:08 +0800 Subject: [PATCH 11/12] addressed ai reviews --- .github/workflows/ci.yml | 5 +- CHANGELOG.md | 8 ++- README.md | 10 +-- src/cli.test.ts | 39 +++++++++++ src/cli.ts | 65 ++++++++++++------- src/config.ts | 9 ++- src/index.ts | 11 ++-- src/server.test.ts | 11 ++++ src/server.ts | 2 +- src/server/config-context.ts | 3 +- src/server/metadata-filter.ts | 9 ++- src/server/query-suggestion.ts | 13 ++-- src/server/reassemble-documents.ts | 2 +- src/server/suggestion-flow.ts | 5 +- src/server/tools/guided-query-tool.ts | 25 ++++--- src/server/tools/suggest-query-params-tool.ts | 2 +- src/server/url-generation.ts | 2 +- 17 files changed, 156 insertions(+), 65 deletions(-) create mode 100644 src/cli.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e728f83..6ae6f8a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,10 +9,11 @@ on: jobs: build-and-test: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest, windows-latest, macos-latest] # Do not add 18.x: Vitest 4 → rolldown imports `styleText` from `node:util` (Node >=20.12 only). # `@vitest/coverage-v8` needs `node:inspector/promises` (Node >=19). Matches `package.json` engines. node-version: [20.x, 22.x] @@ -57,7 +58,7 @@ jobs: path: sbom.cdx.json - name: Upload coverage to Codecov - if: matrix.node-version == '20.x' + if: matrix.os == 'ubuntu-latest' && matrix.node-version == '20.x' uses: codecov/codecov-action@v5 with: files: ./coverage/lcov.info diff --git a/CHANGELOG.md b/CHANGELOG.md index 1452312..fade38e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release ### Added +- `.coderabbit.yaml` sets the pre-merge **docstring coverage** threshold to **79%** (default **80%**) so marginal documentation-only gaps do not block merges; adjust upward as coverage improves. - `registerBuiltinUrlGenerators()` for built-in URL generators; `setupServer()` invokes it so CLI/library parity stays default. - Discriminated result type for `listNamespacesFromKeywordIndex()` (`KeywordIndexNamespacesResult`). - Unit tests for `withRetry` / `withTimeout` in `src/server/retry.test.ts`. @@ -17,14 +18,15 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release - `--version` CLI flag prints the package version and exits. - `list_namespaces` response now includes `expires_at_iso` so clients see the cache expiry as an ISO-8601 timestamp without converting `cache_ttl_seconds`. - `examples/README.md` describing the library embedding sample. -- GitHub Actions **CI** on **Ubuntu** with a **Node.js** matrix (**20.x**, **22.x**): typecheck, lint, Prettier, build, `test:coverage`, **CycloneDX** SBOM artifact upload (per Node version), **Codecov** upload (Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`). +- GitHub Actions **CI** matrix across **ubuntu-latest**, **windows-latest**, and **macos-latest**, each with **Node.js** **20.x** and **22.x**: typecheck, lint, Prettier, build, `test:coverage`, **CycloneDX** SBOM artifact upload (per job), **Codecov** upload (**Ubuntu** + Node **20.x** only), plus a separate **quality** job (`npm audit`, `npm pack --dry-run`). - `npm run test:coverage` with Vitest coverage thresholds (see `vitest.config.ts`). - `@vitest/coverage-v8` devDependency for coverage reports (`lcov`, `json-summary`, HTML). ### Changed +- **Breaking (MCP):** `suggest_query_params` and in-process suggestion flow now emit `recommended_tool` as `count` | `fast` | `detailed` | `full` (aligned with the unified `query` tool `preset`), not legacy `query_fast` / `query_detailed` strings. - **Breaking (MCP):** Single hybrid `query` tool with `preset` (`fast` | `detailed` | `full`); removed separate `query_fast` / `query_detailed` tool registrations. -- **Breaking (library):** Stopped re-exporting `withRetry` / `withTimeout` from the package entry (`server.ts`). +- `resolveConfig()` throws if the Pinecone API key is missing (after trim); library callers must supply `apiKey` via overrides or set `PINECONE_API_KEY`. - `withTimeout` aborts an internal `AbortSignal` on deadline (cooperative cancellation). - `PineconeClient`: shared hit-field extraction, safer merge dedup without empty `_id` collisions, metadata sampling skips zero-vector probe when dimension is unknown, `listNamespacesFromKeywordIndex` surfaces errors via `{ ok: false }`. - Metadata filter manual validation accepts primitive arrays for `$in`/`$nin` including numbers (matches Zod). @@ -32,7 +34,7 @@ Tagged releases are published to npm from GitHub Actions when a **GitHub Release - `.npmignore` no longer excludes `dist/` (still shipped via `package.json` `files`). - `.env.example` log-level options corrected to the four levels actually supported (`DEBUG`, `INFO`, `WARN`, `ERROR`); the stale `WARNING`/`CRITICAL` values are gone. - README Slack URL example now matches the generator output (`https://app.slack.com/client/{team_id}/{channel_id}/p{messageId}`). -- README "Comparison with Python Version" no longer claims an identical API interface; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. +- README "Comparison with Python Version" no longer claims an identical API; the new TypeScript-only tools (`guided_query`, `query_documents`, `keyword_search`, `namespace_router`, `suggest_query_params`, `count`, `generate_urls`) are listed explicitly. - `npm run ci` now runs `test:coverage` so merges are gated on coverage thresholds. - **Breaking (runtime / tooling):** `engines.node` is now **>=20.12.0**. Vitest **4** (bundled **rolldown**) imports `util.styleText` from `node:util` (added in Node **20.12**), and **`@vitest/coverage-v8`** uses `node:inspector/promises` (Node **≥19**). CI tests only **20.x** and **22.x**. - Dependabot groups related **vitest**, **typescript-eslint**, and **eslint/prettier** updates. diff --git a/README.md b/README.md index 090c784..33a4947 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ A Model Context Protocol (MCP) server that provides semantic search over Pinecon - **Semantic Reranking**: Uses BGE reranker model for improved precision - **Dynamic Namespace Discovery**: Automatically discovers available namespaces in your Pinecone index - **Metadata Filtering**: Supports optional metadata filters for refined searches -- **Fast presets**: Lazy initialization, connection pooling, and efficient result merging; use the `query` tool `preset=fast` / `detailed` to trade latency vs quality (no published benchmarks yet — treat descriptions as qualitative). +- **Fast presets**: Lazy initialization, connection pooling, and efficient result merging; use the `query` tool `preset=fast | detailed | full` to trade latency vs quality (no published benchmarks yet — treat descriptions as qualitative). - **Production-oriented defaults**: Input validation, error handling, and configurable logging (semantic versioning is pre-1.0 — review CHANGELOG before upgrading). - **TypeScript Support**: Full TypeScript support with type definitions @@ -260,7 +260,7 @@ Discovers and lists all available namespaces in the configured Pinecone index, i ### `suggest_query_params` -Suggests which **fields** to request and which path to use (`count`, or hybrid query as **fast** vs **detailed** — implemented via the `query` tool `preset` argument), based on the namespace’s schema (from `list_namespaces`) and the user’s natural language query. This is a mandatory flow step before `count` / `query` tools. +Suggests which **fields** to request and which path to use (`count`, or hybrid query presets **fast** / **detailed** / **full** — same vocabulary as the `query` tool `preset` argument), based on the namespace’s schema (from `list_namespaces`) and the user’s natural language query. This is a mandatory flow step before `count` / `query` tools. **Parameters:** @@ -278,7 +278,7 @@ Suggests which **fields** to request and which path to use (`count`, or hybrid q "status": "success", "suggested_fields": ["document_number", "title", "url", "author"], "use_count_tool": false, - "recommended_tool": "query_fast", + "recommended_tool": "fast", "explanation": "User asked for a list or browse; use minimal fields (no chunk_text) for smaller payload and cost.", "namespace_found": true } @@ -292,7 +292,7 @@ Single orchestrator tool that runs the full flow in one call: 1. namespace routing (if namespace is omitted), 2. query param suggestion, -3. execution via `count` or hybrid `query` (fast vs detailed behavior). +3. execution via `count` or hybrid `query` (`fast` / `detailed` / `full` presets). It returns both the final result and a `decision_trace` for transparency. @@ -304,7 +304,7 @@ It returns both the final result and a `decision_trace` for transparency. | `namespace` | string | No | - | Optional explicit namespace | | `metadata_filter` | object | No | - | Optional metadata filter | | `top_k` | integer | No | `10` | Query result size for query paths (1-100) | -| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `fast`, `detailed` | +| `preferred_tool` | enum | No | `auto` | One of `auto`, `count`, `fast`, `detailed`, `full` | | `enrich_urls` | boolean | No | `true` | Auto-generate URLs for `mailing` and `slack-Cpplang` when `metadata.url` is missing | **Returns:** JSON containing `decision_trace` and `result`. diff --git a/src/cli.test.ts b/src/cli.test.ts new file mode 100644 index 0000000..21f6855 --- /dev/null +++ b/src/cli.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; +import { parseCli } from './cli.js'; + +describe('parseCli', () => { + it('does not treat a following flag as a value for --api-key', () => { + const r = parseCli(['--api-key', '--index-name', 'my-index']); + expect(r.kind).toBe('config'); + if (r.kind === 'config') { + expect(r.overrides.apiKey).toBeUndefined(); + expect(r.overrides.indexName).toBe('my-index'); + } + }); + + it('parses --api-key with a real value', () => { + const r = parseCli(['--api-key', 'sk-test', '--index-name', 'idx']); + expect(r.kind).toBe('config'); + if (r.kind === 'config') { + expect(r.overrides.apiKey).toBe('sk-test'); + expect(r.overrides.indexName).toBe('idx'); + } + }); + + it('does not consume --top-k when the next token is another flag', () => { + const r = parseCli(['--top-k', '--log-level', 'DEBUG']); + expect(r.kind).toBe('config'); + if (r.kind === 'config') { + expect(r.overrides.defaultTopK).toBeUndefined(); + expect(r.overrides.logLevel).toBe('DEBUG'); + } + }); + + it('parses numeric --top-k when value is valid', () => { + const r = parseCli(['--top-k', '25']); + expect(r.kind).toBe('config'); + if (r.kind === 'config') { + expect(r.overrides.defaultTopK).toBe(25); + } + }); +}); diff --git a/src/cli.ts b/src/cli.ts index fd7d7af..6123a23 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -17,13 +17,19 @@ function parsePositiveInt(raw: string | undefined): number | undefined { return Number.isFinite(n) && n > 0 ? n : undefined; } +/** Next argv token if it is a real value (not another flag). */ +function readOptionValue(argv: string[], i: number): string | undefined { + const v = argv[i + 1]; + if (v === undefined || v.startsWith('-')) return undefined; + return v; +} + /** Parse `process.argv` (slice 2..) into help/version/config result. */ export function parseCli(argv: string[] = process.argv.slice(2)): ParseCliResult { const overrides: ConfigOverrides = {}; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; - const next = argv[i + 1]; switch (arg) { case '--help': @@ -32,52 +38,66 @@ export function parseCli(argv: string[] = process.argv.slice(2)): ParseCliResult case '--version': case '-v': return { kind: 'version' }; - case '--api-key': - if (next !== undefined) { - overrides.apiKey = next; + case '--api-key': { + const v = readOptionValue(argv, i); + if (v !== undefined) { + overrides.apiKey = v; i++; } break; - case '--index-name': - if (next !== undefined) { - overrides.indexName = next; + } + case '--index-name': { + const v = readOptionValue(argv, i); + if (v !== undefined) { + overrides.indexName = v; i++; } break; - case '--sparse-index-name': - if (next !== undefined) { - overrides.sparseIndexName = next; + } + case '--sparse-index-name': { + const v = readOptionValue(argv, i); + if (v !== undefined) { + overrides.sparseIndexName = v; i++; } break; - case '--rerank-model': - if (next !== undefined) { - overrides.rerankModel = next; + } + case '--rerank-model': { + const v = readOptionValue(argv, i); + if (v !== undefined) { + overrides.rerankModel = v; i++; } break; + } case '--top-k': { - const n = parsePositiveInt(next); + const raw = readOptionValue(argv, i); + const n = parsePositiveInt(raw); if (n !== undefined) { overrides.defaultTopK = n; i++; } break; } - case '--log-level': - if (next !== undefined) { - overrides.logLevel = next; + case '--log-level': { + const v = readOptionValue(argv, i); + if (v !== undefined) { + overrides.logLevel = v; i++; } break; - case '--log-format': - if (next !== undefined) { - overrides.logFormat = next; + } + case '--log-format': { + const v = readOptionValue(argv, i); + if (v !== undefined) { + overrides.logFormat = v; i++; } break; + } case '--cache-ttl-seconds': { - const n = parsePositiveInt(next); + const raw = readOptionValue(argv, i); + const n = parsePositiveInt(raw); if (n !== undefined) { overrides.cacheTtlSeconds = n; i++; @@ -85,7 +105,8 @@ export function parseCli(argv: string[] = process.argv.slice(2)): ParseCliResult break; } case '--request-timeout-ms': { - const n = parsePositiveInt(next); + const raw = readOptionValue(argv, i); + const n = parsePositiveInt(raw); if (n !== undefined) { overrides.requestTimeoutMs = n; i++; diff --git a/src/config.ts b/src/config.ts index 828cc3e..11b2fc4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -93,12 +93,19 @@ export interface ConfigOverrides { /** * Build a `ServerConfig` from CLI overrides, environment variables, and defaults. * CLI > env > default precedence is preserved. + * + * @throws Error when no API key is provided (empty after trim from overrides or `PINECONE_API_KEY`). */ export function resolveConfig( overrides: ConfigOverrides, env: NodeJS.ProcessEnv = process.env ): ServerConfig { - const apiKey = overrides.apiKey ?? env['PINECONE_API_KEY'] ?? ''; + const apiKey = (overrides.apiKey ?? env['PINECONE_API_KEY'] ?? '').trim(); + if (!apiKey) { + throw new Error( + 'Missing Pinecone API key: set PINECONE_API_KEY or pass --api-key (or apiKey in ConfigOverrides for library use).' + ); + } const indexName = overrides.indexName ?? env['PINECONE_INDEX_NAME'] ?? DEFAULT_INDEX_NAME; const sparseIndexName = overrides.sparseIndexName ?? env['PINECONE_SPARSE_INDEX_NAME'] ?? `${indexName}-sparse`; diff --git a/src/index.ts b/src/index.ts index ef26a78..1050743 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,14 +33,13 @@ function buildConfigOrExit(): ServerConfig { process.exit(0); } - const config = resolveConfig(parsed.overrides); - if (!config.apiKey) { - process.stderr.write( - 'Error: Pinecone API key is required. Set PINECONE_API_KEY environment variable or use --api-key option.\n' - ); + try { + return resolveConfig(parsed.overrides); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + process.stderr.write(`Error: ${message}\n`); process.exit(1); } - return config; } async function main(): Promise { diff --git a/src/server.test.ts b/src/server.test.ts index 281a257..602e98b 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -13,6 +13,7 @@ describe('suggestQueryParams', () => { it('suggests count tool and minimal fields for count-style queries', () => { const r = suggestQueryParams(wg21Fields, 'How many papers by John Doe?'); + expect(r.recommended_tool).toBe('count'); expect(r.use_count_tool).toBe(true); expect(r.suggested_fields).toContain('document_number'); expect(r.suggested_fields).toContain('url'); @@ -21,6 +22,7 @@ describe('suggestQueryParams', () => { it('suggests chunk_text for content-style queries', () => { const r = suggestQueryParams(wg21Fields, 'What does the paper say about contracts?'); + expect(r.recommended_tool).toBe('detailed'); expect(r.use_count_tool).toBe(false); expect(r.suggested_fields).toContain('chunk_text'); expect(r.namespace_found).toBe(true); @@ -28,6 +30,7 @@ describe('suggestQueryParams', () => { it('suggests minimal fields for list-style queries', () => { const r = suggestQueryParams(wg21Fields, 'List papers by Michael Wong with titles and links'); + expect(r.recommended_tool).toBe('fast'); expect(r.use_count_tool).toBe(false); expect(r.suggested_fields).not.toContain('chunk_text'); expect(r.suggested_fields).toContain('title'); @@ -37,6 +40,7 @@ describe('suggestQueryParams', () => { it('returns namespace_found false when metadata is null', () => { const r = suggestQueryParams(null, 'list papers'); + expect(r.recommended_tool).toBe('fast'); expect(r.namespace_found).toBe(false); expect(r.suggested_fields).toEqual([]); }); @@ -62,6 +66,13 @@ describe('validateMetadataFilter', () => { expect(result).toBeNull(); }); + it('accepts boolean arrays for $in', () => { + const result = validateMetadataFilter({ + active: { $in: [true, false] }, + }); + expect(result).toBeNull(); + }); + it('rejects unsupported operators', () => { const result = validateMetadataFilter({ year: { $regex: '^202' }, diff --git a/src/server.ts b/src/server.ts index 3644524..ca24c86 100644 --- a/src/server.ts +++ b/src/server.ts @@ -37,7 +37,7 @@ export { setPineconeClient } from './server/client-context.js'; export { validateMetadataFilter } from './server/metadata-filter.js'; /** Heuristic field + tool suggestions from a namespace schema + user query. */ export { suggestQueryParams } from './server/query-suggestion.js'; -export type { SuggestQueryParamsResult } from './server/query-suggestion.js'; +export type { RecommendedTool, SuggestQueryParamsResult } from './server/query-suggestion.js'; /** Register custom per-namespace URL synthesis used by `generate_urls` / row enrichment. */ export { registerUrlGenerator, diff --git a/src/server/config-context.ts b/src/server/config-context.ts index 281c1e4..763ac94 100644 --- a/src/server/config-context.ts +++ b/src/server/config-context.ts @@ -13,7 +13,8 @@ export function setServerConfig(config: ServerConfig): void { * (namespace cache TTL, suggest-flow gate, etc.). * * When `setupServer()` runs without an explicit config, falls back to `resolveConfig({})` - * so env defaults still apply. + * so env defaults still apply. That requires `PINECONE_API_KEY` (or the call throws); embedders + * should pass `config` into `setupServer(config)` when env is not set. */ export function getServerConfig(): ServerConfig { if (!activeConfig) { diff --git a/src/server/metadata-filter.ts b/src/server/metadata-filter.ts index a323b29..cfcf281 100644 --- a/src/server/metadata-filter.ts +++ b/src/server/metadata-filter.ts @@ -1,14 +1,13 @@ import { z } from 'zod'; +const primitiveScalarSchema = z.union([z.string(), z.number(), z.boolean()]); + // Recursive Zod schema for Pinecone metadata filters // Supports nested objects with operators like {"timestamp": {"$gte": 123}} const metadataFilterValueSchema: z.ZodType = z.lazy(() => z.union([ - z.string(), - z.number(), - z.boolean(), - z.array(z.string()), - z.array(z.number()), + primitiveScalarSchema, + z.array(primitiveScalarSchema), z.array(z.lazy(() => metadataFilterSchema)), z.record(z.string(), metadataFilterValueSchema), // Recursive for nested operators ]) diff --git a/src/server/query-suggestion.ts b/src/server/query-suggestion.ts index 8d1ed7e..f02dca8 100644 --- a/src/server/query-suggestion.ts +++ b/src/server/query-suggestion.ts @@ -1,10 +1,13 @@ import { COUNT_FIELDS } from '../constants.js'; +/** Values returned by `suggest_query_params` / suggestion flow; align with `query` tool `preset`. */ +export type RecommendedTool = 'count' | 'fast' | 'detailed' | 'full'; + /** Suggested query params from namespace schema and user query. */ export type SuggestQueryParamsResult = { suggested_fields: string[]; use_count_tool: boolean; - recommended_tool: 'count' | 'query_fast' | 'query_detailed'; + recommended_tool: RecommendedTool; explanation: string; namespace_found: boolean; }; @@ -25,7 +28,7 @@ export function suggestQueryParams( return { suggested_fields: [], use_count_tool: false, - recommended_tool: 'query_fast', + recommended_tool: 'fast', explanation: 'Namespace not found or has no metadata fields. Call list_namespaces first, then pass a valid namespace.', namespace_found: false, @@ -35,7 +38,7 @@ export function suggestQueryParams( return { suggested_fields: [], use_count_tool: false, - recommended_tool: 'query_fast', + recommended_tool: 'fast', explanation: 'Namespace has no metadata fields. Use list_namespaces to verify the namespace is correct.', namespace_found: true, @@ -63,7 +66,7 @@ export function suggestQueryParams( return { suggested_fields: fields.length ? fields : available, use_count_tool: false, - recommended_tool: 'query_detailed', + recommended_tool: 'detailed', explanation: 'User asked for content or details; include chunk_text for snippets.', namespace_found: true, }; @@ -74,7 +77,7 @@ export function suggestQueryParams( return { suggested_fields: listFields.length ? listFields : available.slice(0, 5), use_count_tool: false, - recommended_tool: 'query_fast', + recommended_tool: 'fast', explanation: 'User asked for a list or browse; use minimal fields (no chunk_text) for smaller payload and cost.', namespace_found: true, diff --git a/src/server/reassemble-documents.ts b/src/server/reassemble-documents.ts index c66590e..ef900dd 100644 --- a/src/server/reassemble-documents.ts +++ b/src/server/reassemble-documents.ts @@ -50,7 +50,7 @@ export interface ReassembledDocument { /** * Group search results by document and merge chunk content. * Chunks are ordered by metadata chunk_index (or similar) when present; otherwise retrieval order. - * * + * * Document identity is derived in priority order: document_number → url → doc_id → hit.id. * Chunks whose metadata contains none of the first three keys are grouped by their raw vector ID, * so each such chunk becomes its own single-chunk "document" in the output. diff --git a/src/server/suggestion-flow.ts b/src/server/suggestion-flow.ts index e8ddbcb..2f77d13 100644 --- a/src/server/suggestion-flow.ts +++ b/src/server/suggestion-flow.ts @@ -1,8 +1,9 @@ import { getServerConfig } from './config-context.js'; +import type { RecommendedTool } from './query-suggestion.js'; type FlowState = { updatedAt: number; - recommended_tool: 'count' | 'query_fast' | 'query_detailed'; + recommended_tool: RecommendedTool; suggested_fields: string[]; user_query: string; }; @@ -55,7 +56,7 @@ export function requireSuggested(namespace: string): ok: true, flow: { updatedAt: Date.now(), - recommended_tool: 'query_fast', + recommended_tool: 'fast', suggested_fields: [], user_query: '', }, diff --git a/src/server/tools/guided-query-tool.ts b/src/server/tools/guided-query-tool.ts index 08cdf29..fa31a3e 100644 --- a/src/server/tools/guided-query-tool.ts +++ b/src/server/tools/guided-query-tool.ts @@ -12,16 +12,17 @@ import { markSuggested } from '../suggestion-flow.js'; import { getToolErrorMessage, logToolError } from '../tool-error.js'; import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; -type GuidedToolName = 'count' | 'query_fast' | 'query_detailed'; +type GuidedToolName = 'count' | 'fast' | 'detailed' | 'full'; function resolveGuidedToolName( - preferred: 'auto' | 'count' | 'fast' | 'detailed', + preferred: 'auto' | 'count' | 'fast' | 'detailed' | 'full', suggestion: { recommended_tool: GuidedToolName } ): GuidedToolName { if (preferred === 'auto') return suggestion.recommended_tool; if (preferred === 'count') return 'count'; - if (preferred === 'fast') return 'query_fast'; - return 'query_detailed'; + if (preferred === 'fast') return 'fast'; + if (preferred === 'detailed') return 'detailed'; + return 'full'; } /** Register the guided_query orchestrator tool on the MCP server. */ @@ -31,7 +32,7 @@ export function registerGuidedQueryTool(server: McpServer): void { { description: 'Single orchestrator that runs routing + suggestion + execution in one call. ' + - 'Flow: optional namespace_router logic -> suggest_query_params logic -> executes count or hybrid query (fast vs detailed preset). ' + + 'Flow: optional namespace_router logic -> suggest_query_params logic -> executes count or hybrid query (fast / detailed / full presets). ' + 'Returns decision_trace so behavior stays transparent and debuggable.', inputSchema: { user_query: z.string().describe('User question or intent.'), @@ -52,10 +53,10 @@ export function registerGuidedQueryTool(server: McpServer): void { .default(10) .describe('Result count for hybrid query paths (1-100).'), preferred_tool: z - .enum(['auto', 'count', 'fast', 'detailed']) + .enum(['auto', 'count', 'fast', 'detailed', 'full']) .default('auto') .describe( - 'Optional override: count, fast (no rerank / light fields), detailed (reranked), or auto from suggestion.' + 'Optional override: count, fast (no rerank / light fields), detailed (reranked), full (explicit rerank + fields), or auto from suggestion.' ), enrich_urls: z .boolean() @@ -149,7 +150,13 @@ export function registerGuidedQueryTool(server: McpServer): void { }); } - const isFast = selectedTool === 'query_fast'; + const isFast = selectedTool === 'fast'; + const mode: QueryResponse['mode'] = + selectedTool === 'fast' + ? 'query_fast' + : selectedTool === 'detailed' + ? 'query_detailed' + : 'query'; const fields = suggestion.suggested_fields.length > 0 ? suggestion.suggested_fields @@ -170,7 +177,7 @@ export function registerGuidedQueryTool(server: McpServer): void { }); const result: QueryResponse = { status: 'success', - mode: isFast ? 'query_fast' : 'query_detailed', + mode, query: queryText, namespace, metadata_filter: metadata_filter, diff --git a/src/server/tools/suggest-query-params-tool.ts b/src/server/tools/suggest-query-params-tool.ts index d196490..515dae5 100644 --- a/src/server/tools/suggest-query-params-tool.ts +++ b/src/server/tools/suggest-query-params-tool.ts @@ -14,7 +14,7 @@ export function registerSuggestQueryParamsTool(server: McpServer): void { description: "Suggest which fields to request and whether to use the count tool, based on the namespace schema (from list_namespaces) and the user's natural language query. " + 'Call list_namespaces first to get available namespaces and metadata fields. Then call this tool with the target namespace and the user query; ' + - 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count / query_fast / query_detailed — map query_fast→query preset fast, query_detailed→preset detailed), and an explanation. ' + + 'it returns suggested_fields (only fields that exist in that namespace), use_count_tool (true if the query is a count question), recommended_tool (count | fast | detailed | full — same vocabulary as the query tool preset), and an explanation. ' + 'This step is mandatory before query/count tools; use the returned suggested_fields in query tools to reduce payload and cost.', inputSchema: { namespace: z diff --git a/src/server/url-generation.ts b/src/server/url-generation.ts index e3ee393..b4e20a1 100644 --- a/src/server/url-generation.ts +++ b/src/server/url-generation.ts @@ -101,9 +101,9 @@ let builtinGeneratorsRegistered = false; */ export function registerBuiltinUrlGenerators(): void { if (builtinGeneratorsRegistered) return; - builtinGeneratorsRegistered = true; urlGenerators.set('mailing', generatorMailing); urlGenerators.set('slack-Cpplang', generatorSlackCpplang); + builtinGeneratorsRegistered = true; } /** From b56ed2e584fe951a7f21eb0936ada6013b1f7c5a Mon Sep 17 00:00:00 2001 From: zho Date: Thu, 14 May 2026 03:39:19 +0800 Subject: [PATCH 12/12] fixed ci error --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6ae6f8a..33681d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: - name: Upload SBOM artifact uses: actions/upload-artifact@v4 with: - name: sbom-cyclonedx-node-${{ matrix.node-version }} + name: sbom-cyclonedx-${{ matrix.os }}-node-${{ matrix.node-version }} path: sbom.cdx.json - name: Upload coverage to Codecov