diff --git a/README.md b/README.md index 668bf24..c5169c4 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,12 @@ The server requires a Pinecone API key and supports the following configuration ### Environment Variables -| Variable | Required | Default | Description | -| ---------------------------------- | -------- | -------------------- | --------------------- | -| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | -| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name | -| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | -| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | +| Variable | Required | Default | Description | +| ---------------------------------- | -------- | ---------------------- | ------------------------------------------------ | +| `PINECONE_API_KEY` | Yes | - | Your Pinecone API key | +| `PINECONE_INDEX_NAME` | No | `rag-hybrid` | Pinecone index name (dense + sparse for hybrid) | +| `PINECONE_RERANK_MODEL` | No | `bge-reranker-v2-m3` | Reranking model | +| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No | `INFO` | Logging level | ### Claude Desktop Configuration @@ -143,11 +143,11 @@ node node_modules/@will-cppa/pinecone-read-only-mcp/dist/index.js --api-key YOUR ### Available Options ``` ---api-key TEXT Pinecone API key (or set PINECONE_API_KEY env var) ---index-name TEXT Pinecone index name [default: rag-hybrid] ---rerank-model TEXT Reranking model [default: bge-reranker-v2-m3] ---log-level TEXT Logging level [default: INFO] ---help, -h Show help message +--api-key TEXT Pinecone API key (or set PINECONE_API_KEY env var) +--index-name TEXT Pinecone index name [default: rag-hybrid] +--rerank-model TEXT Reranking model [default: bge-reranker-v2-m3] +--log-level TEXT Logging level [default: INFO] +--help, -h Show help message ``` ## Deployment @@ -338,6 +338,45 @@ Returns the **unique document count** matching a metadata filter and semantic qu } ``` +### `keyword_search` + +Performs **keyword (lexical/sparse-only)** search over the dedicated sparse index (default: `rag-hybrid-sparse`, i.e. `{PINECONE_INDEX_NAME}-sparse`). Use for exact or keyword-style queries. Does not use the dense index or semantic reranking. Call `list_namespaces` first to discover namespaces; `suggest_query_params` is optional. + +**Parameters:** + +| Parameter | Type | Required | Default | Description | +| ----------------- | -------- | -------- | ------- | --------------------------------------------------------------------------- | +| `query_text` | string | Yes | - | Search query text (keyword/lexical match) | +| `namespace` | string | Yes | - | Namespace to search (use `list_namespaces` to discover) | +| `top_k` | integer | No | `10` | Number of results to return (1-100) | +| `metadata_filter` | object | No | - | Optional metadata filter (same operators as `query`) | +| `fields` | string[] | No | - | Optional field names to return; omit for all fields | + +**Returns:** JSON with `status`, `query`, `namespace`, `index` (sparse index name), `result_count`, and `results` (ids, metadata, scores). Result rows match the `query` tool shape (e.g. `paper_number`, `title`, `author`, `url`, `content`, `score`, `reranked: false`). + +**Example response:** + +```json +{ + "status": "success", + "query": "contracts C++", + "namespace": "wg21-papers", + "index": "rag-hybrid-sparse", + "result_count": 5, + "results": [ + { + "paper_number": "P0548", + "title": "Contracts for C++", + "author": "John Doe", + "url": "https://...", + "content": "...", + "score": 0.85, + "reranked": false + } + ] +} +``` + ### `query` Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. For **count** questions, use the `count` tool instead. @@ -470,6 +509,18 @@ npm run build npm test ``` +### Testing the keyword_search tool + +1. **Connectivity and keyword search (script):** + Run the search test script (includes a keyword search step against the sparse index): + ```bash + PINECONE_API_KEY=your-key npm run test:search + ``` + If the sparse index (`rag-hybrid-sparse` by default) does not exist or has no data, the keyword search step is skipped with a warning. + +2. **Via MCP client:** + Start the server and call the `keyword_search` tool with `query_text`, `namespace` (from `list_namespaces`), and optional `top_k` or `metadata_filter`. Response shape is the same as the `query` tool (e.g. `results` with ids, metadata, scores; `reranked` is always `false`). + ### Code Quality ```bash diff --git a/scripts/test-search.ts b/scripts/test-search.ts index b486847..8623369 100644 --- a/scripts/test-search.ts +++ b/scripts/test-search.ts @@ -152,6 +152,50 @@ async function test() { ); } + // Test 5: Keyword (sparse-only) search — use namespace from sparse index, not dense + let duration5: number | undefined; + let test5Skipped = false; + const sparseNamespaces = await client.listNamespacesFromKeywordIndex(); + if (sparseNamespaces.length === 0) { + test5Skipped = true; + console.log(`\n🔤 Test 5: Keyword search (sparse-only index)`); + console.log( + `⚠️ Keyword search skipped: sparse index has no namespaces (or index unavailable).` + ); + console.log( + ` Ensure the sparse index (e.g. rag-hybrid-sparse) exists and has data.` + ); + } else { + const sparseTestNamespace = sparseNamespaces[0].namespace; + console.log(`\n🔤 Test 5: Keyword search (sparse-only index)`); + console.log(` Namespace: "${sparseTestNamespace}" (from sparse index)`); + console.log(` Query: "test query"`); + console.log(` Top K: 3`); + try { + const startTime5 = Date.now(); + const results5 = await client.keywordSearch({ + query: 'test query', + namespace: sparseTestNamespace, + topK: 3, + }); + duration5 = Date.now() - startTime5; + console.log(`✅ Keyword search returned ${results5.length} result(s) in ${duration5}ms`); + if (results5.length > 0) { + console.log( + ` First result score: ${results5[0].score.toFixed(4)}, reranked: ${results5[0].reranked}` + ); + } + } catch (kwError) { + test5Skipped = true; + console.log( + `⚠️ Keyword search skipped: ${kwError instanceof Error ? kwError.message : String(kwError)}` + ); + console.log( + ` Ensure the sparse index exists and namespace "${sparseTestNamespace}" has data.` + ); + } + } + console.log('\n✨ All tests completed successfully!'); console.log(`\nPerformance comparison:`); console.log(` Without reranking: ${duration1}ms`); @@ -159,6 +203,11 @@ async function test() { if (duration3 !== undefined) { console.log(` With metadata filter: ${duration3}ms`); } + if (duration5 !== undefined) { + console.log(` Keyword search: ${duration5}ms`); + } else if (test5Skipped) { + console.log(` Keyword search: skipped`); + } console.log(` Reranking overhead: ${duration2 - duration1}ms`); } catch (error) { console.error('\n❌ Error during testing:', error); diff --git a/src/constants.ts b/src/constants.ts index 0aa7ade..01122ab 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -45,6 +45,7 @@ Features: - Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count. - URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url. - Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization. +- Keyword search: Use keyword_search to query the sparse index (default: rag-hybrid-sparse) for lexical/keyword-only retrieval without reranking. Usage: 1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index diff --git a/src/index.ts b/src/index.ts index 0b9163c..959a7fe 100644 --- a/src/index.ts +++ b/src/index.ts @@ -71,15 +71,15 @@ Pinecone Read-Only MCP Server Usage: pinecone-read-only-mcp [options] Options: - --api-key TEXT Pinecone API key (or set PINECONE_API_KEY env var) - --index-name TEXT Pinecone index name [default: ${DEFAULT_INDEX_NAME}] - --rerank-model TEXT Reranking model [default: ${DEFAULT_RERANK_MODEL}] - --log-level TEXT Logging level [default: INFO] - --help, -h Show this help message + --api-key TEXT Pinecone API key (or set PINECONE_API_KEY env var) + --index-name TEXT Pinecone index name [default: ${DEFAULT_INDEX_NAME}]; sparse index is {index-name}-sparse + --rerank-model TEXT Reranking model [default: ${DEFAULT_RERANK_MODEL}] + --log-level TEXT Logging level [default: INFO] + --help, -h Show this help message Environment Variables: PINECONE_API_KEY Pinecone API key - PINECONE_INDEX_NAME Pinecone index name + PINECONE_INDEX_NAME Pinecone index name (sparse index: {name}-sparse) PINECONE_RERANK_MODEL Reranking model name PINECONE_READ_ONLY_MCP_LOG_LEVEL Logging level @@ -131,7 +131,7 @@ async function main(): Promise { setPineconeClient(client); console.error(`Starting Pinecone Read-Only MCP server with stdio transport`); - console.error(`Using Pinecone index: ${indexName}`); + console.error(`Using Pinecone index: ${indexName} (sparse: ${indexName}-sparse)`); console.error(`Log level: ${logLevel}`); // Setup server diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts index 4980880..4b7b1ef 100644 --- a/src/pinecone-client.ts +++ b/src/pinecone-client.ts @@ -20,6 +20,7 @@ import type { QueryParams, CountParams, CountResult, + KeywordSearchParams, MergedHit, NamespaceHandle, SearchableIndex, @@ -74,6 +75,28 @@ export class PineconeClient { config.defaultTopK || parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K)); } + /** Returns the sparse index name (same as hybrid sparse: {indexName}-sparse). Used for keyword_search response. */ + getSparseIndexName(): string { + return `${this.indexName}-sparse`; + } + + /** + * Normalize and clamp topK from request (validates >= 1, caps at MAX_TOP_K). + */ + private clampTopK(requested: number | undefined): number { + if (requested !== undefined && !Number.isFinite(requested)) { + throw new Error('topK must be a finite number >= 1'); + } + let topK = requested !== undefined ? requested : this.defaultTopK; + if (topK < 1) { + throw new Error('topK must be at least 1'); + } + if (topK > MAX_TOP_K) { + topK = MAX_TOP_K; + } + return topK; + } + /** * Ensure Pinecone client is initialized */ @@ -103,7 +126,7 @@ export class PineconeClient { const pc = this.ensureClient(); const denseName = this.indexName; - const sparseName = `${this.indexName}-sparse`; + const sparseName = this.getSparseIndexName(); const dense = pc.index(denseName) as unknown as SearchableIndex; const sparse = pc.index(sparseName) as unknown as SearchableIndex; @@ -115,6 +138,29 @@ export class PineconeClient { return { denseIndex: dense, sparseIndex: sparse }; } + /** + * List namespaces present on the sparse index (same index used for hybrid sparse and keyword_search). + * Use this to choose a namespace for sparse-only queries instead of the dense index list. + */ + async listNamespacesFromKeywordIndex(): Promise< + Array<{ namespace: string; recordCount: number }> + > { + try { + const { sparseIndex } = await this.ensureIndexes(); + const stats = sparseIndex.describeIndexStats + ? await sparseIndex.describeIndexStats() + : undefined; + const namespaces = stats?.namespaces ?? {}; + return Object.entries(namespaces).map(([namespace, info]) => ({ + namespace, + recordCount: info?.recordCount ?? 0, + })); + } catch (error) { + logError('Error listing namespaces from keyword index', error); + return []; + } + } + /** * List all available namespaces with their metadata information * @@ -390,14 +436,7 @@ export class PineconeClient { throw new Error('Query cannot be empty'); } - let topK = requestedTopK !== undefined ? requestedTopK : this.defaultTopK; - if (topK < 1) { - throw new Error('topK must be at least 1'); - } - - if (topK > MAX_TOP_K) { - topK = MAX_TOP_K; - } + const topK = this.clampTopK(requestedTopK); // When reranking, Pinecone requires chunk_text in returned fields; add it if user specified fields without it const searchFields = @@ -453,6 +492,64 @@ export class PineconeClient { return documents; } + /** + * Keyword (sparse-only) search against the dedicated sparse index. + * Performs lexical/keyword retrieval only—no dense index, no reranking. + * Use for exact or keyword-style queries on the configured sparse index. + */ + async keywordSearch(params: KeywordSearchParams): Promise { + const { + query, + namespace, + topK: requestedTopK, + metadataFilter, + fields: requestedFields, + } = params; + + if (!query || !query.trim()) { + throw new Error('Query cannot be empty'); + } + + const topK = this.clampTopK(requestedTopK); + + const { sparseIndex } = await this.ensureIndexes(); + const searchOptions = requestedFields?.length ? { fields: requestedFields } : undefined; + + const hits = await this.searchIndex( + sparseIndex, + query.trim(), + topK, + namespace, + metadataFilter, + searchOptions + ); + + const documents: SearchResult[] = hits.map((hit) => { + const fields = hit.fields || {}; + let content = ''; + const metadata: Record = {}; + for (const [key, value] of Object.entries(fields)) { + if (key === 'chunk_text') { + content = typeof value === 'string' ? value : ''; + } else { + metadata[key] = value as PineconeMetadataValue; + } + } + return { + id: hit._id || '', + content, + score: hit._score || 0, + metadata, + reranked: false, + }; + }); + + logInfo( + `Keyword search returned ${documents.length} results from ${this.getSparseIndexName()}` + ); + return documents; + } + /** * Return the number of unique documents matching the query and optional metadata filter. * Uses semantic search only (dense index), requests minimal fields (document_number, url, doc_id) diff --git a/src/server.ts b/src/server.ts index a9beb82..3459677 100644 --- a/src/server.ts +++ b/src/server.ts @@ -10,6 +10,7 @@ import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js import { registerCountTool } from './server/tools/count-tool.js'; import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js'; import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js'; +import { registerKeywordSearchTool } from './server/tools/keyword-search-tool.js'; import { registerListNamespacesTool } from './server/tools/list-namespaces-tool.js'; import { registerNamespaceRouterTool } from './server/tools/namespace-router-tool.js'; import { registerQueryDocumentsTool } from './server/tools/query-documents-tool.js'; @@ -38,6 +39,7 @@ export async function setupServer(): Promise { registerSuggestQueryParamsTool(server); registerCountTool(server); registerQueryTool(server); + registerKeywordSearchTool(server); registerQueryDocumentsTool(server); registerGuidedQueryTool(server); registerGenerateUrlsTool(server); diff --git a/src/server/tools/keyword-search-tool.ts b/src/server/tools/keyword-search-tool.ts new file mode 100644 index 0000000..1d6c257 --- /dev/null +++ b/src/server/tools/keyword-search-tool.ts @@ -0,0 +1,148 @@ +import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { z } from 'zod'; +import { MAX_TOP_K, MIN_TOP_K } from '../../constants.js'; +import { getPineconeClient } from '../client-context.js'; +import { formatQueryResultRows } from '../format-query-result.js'; +import { metadataFilterSchema, validateMetadataFilter } from '../metadata-filter.js'; +import { getToolErrorMessage, logToolError } from '../tool-error.js'; +import { jsonErrorResponse, jsonResponse } from '../tool-response.js'; + +/** Response shape for keyword_search: same as query tool for consistency. */ +export interface KeywordSearchResponse { + status: 'success' | 'error'; + query?: string; + namespace?: string; + index?: string; + metadata_filter?: Record; + result_count?: number; + results?: Array<{ + paper_number: string | null; + title: string; + author: string; + url: string; + content: string; + score: number; + reranked: boolean; + metadata?: Record; + }>; + fields?: string[]; + message?: string; +} + +async function executeKeywordSearch(params: { + query_text: string; + namespace: string; + top_k: number; + metadata_filter?: Record; + fields?: string[]; +}): Promise { + const { query_text, namespace, top_k, metadata_filter, fields } = params; + + const normalizedQuery = query_text.trim(); + const normalizedNamespace = namespace?.trim() ?? ''; + + if (!normalizedQuery) { + return { + status: 'error', + message: 'Query text cannot be empty', + }; + } + + if (!normalizedNamespace) { + return { + status: 'error', + message: 'Namespace cannot be empty', + }; + } + + if (metadata_filter) { + const filterError = validateMetadataFilter(metadata_filter); + if (filterError) { + return { status: 'error', message: filterError }; + } + } + + const client = getPineconeClient(); + const results = await client.keywordSearch({ + query: normalizedQuery, + namespace: normalizedNamespace, + topK: top_k, + metadataFilter: metadata_filter, + fields: fields?.length ? fields : undefined, + }); + + const formattedResults = formatQueryResultRows(results, { + namespace: normalizedNamespace, + }); + + const response: KeywordSearchResponse = { + status: 'success', + query: normalizedQuery, + namespace: normalizedNamespace, + index: client.getSparseIndexName(), + metadata_filter: metadata_filter, + result_count: formattedResults.length, + results: formattedResults, + }; + if (fields?.length) { + response.fields = fields; + } + return response; +} + +/** Register the keyword_search tool on the MCP server. */ +export function registerKeywordSearchTool(server: McpServer): void { + server.registerTool( + 'keyword_search', + { + description: + 'Keyword (lexical/sparse-only) search over the Pinecone sparse index (default: rag-hybrid-sparse). ' + + 'Use for exact or keyword-style queries. Does not use semantic reranking. ' + + 'Call list_namespaces first to discover namespaces; suggest_query_params is optional.', + inputSchema: { + query_text: z.string().describe('Search query text (keyword/lexical match).'), + namespace: z + .string() + .describe('Namespace to search. Use list_namespaces to discover available namespaces.'), + top_k: z + .number() + .int() + .min(MIN_TOP_K) + .max(MAX_TOP_K) + .default(10) + .describe('Number of results to return (1-100). Default: 10'), + metadata_filter: metadataFilterSchema + .optional() + .describe('Optional metadata filter to narrow results.'), + fields: z + .array(z.string()) + .optional() + .describe( + 'Optional field names to return. Omit for all fields; use suggest_query_params for suggestions.' + ), + }, + }, + async (params) => { + try { + const response = await executeKeywordSearch({ + query_text: params.query_text, + namespace: params.namespace, + top_k: params.top_k, + metadata_filter: params.metadata_filter, + fields: params.fields, + }); + if (response.status === 'error') { + return jsonErrorResponse(response); + } + return jsonResponse(response); + } catch (error) { + logToolError('keyword_search', error); + const response: KeywordSearchResponse = { + status: 'error', + message: getToolErrorMessage(error, 'Keyword search failed'), + }; + return jsonErrorResponse(response); + } + } + ); +} diff --git a/src/types.ts b/src/types.ts index 485e8a1..b737265 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,6 +53,16 @@ export interface CountParams { metadataFilter?: Record; } +/** Parameters for keyword (sparse-only) search against the dedicated sparse index. */ +export interface KeywordSearchParams { + query: string; + namespace: string; + topK?: number; + metadataFilter?: Record; + /** If set, only these fields are returned. Omit for all fields. */ + fields?: string[]; +} + /** Result of a count request: unique document count (deduped by doc id/url); truncated when at least COUNT_TOP_K. */ export interface CountResult { count: number;