feat: add keyword_search tool for pinecone-rag-sparse index (#29)

feat: add keyword_search tool for pinecone-rag-sparse index (#29)

- Add PineconeClient.keywordSearch() for sparse-only (lexical) search
- Register keyword_search MCP tool (query_text, namespace, top_k, optional metadata_filter, fields)
- Config: PINECONE_SPARSE_INDEX_NAME / --sparse-index-name (default: pinecone-rag-sparse)
- Document tool in README; add Test 5 in scripts/test-search.ts
- No reranking; returns same result shape as query tool (reranked: false)

Author: clean6378-max-it

README.md

@@ -58,12 +58,13 @@ The server requires a Pinecone API key and supports the following configuration
 
 ### Environment Variables
 
-| Variable                           | Required | Default              | Description           |
-| ---------------------------------- | -------- | -------------------- | --------------------- |
-| `PINECONE_API_KEY`                 | Yes      | -                    | Your Pinecone API key |
-| `PINECONE_INDEX_NAME`              | No       | `rag-hybrid`         | Pinecone index name   |
-| `PINECONE_RERANK_MODEL`            | No       | `bge-reranker-v2-m3` | Reranking model       |
-| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No       | `INFO`               | Logging level         |
+| Variable                           | Required | Default                | Description                                      |
+| ---------------------------------- | -------- | ---------------------- | ------------------------------------------------ |
+| `PINECONE_API_KEY`                 | Yes      | -                      | Your Pinecone API key                            |
+| `PINECONE_INDEX_NAME`              | No       | `rag-hybrid`           | Pinecone index name (dense + sparse for hybrid)   |
+| `PINECONE_SPARSE_INDEX_NAME`       | No       | `pinecone-rag-sparse`  | Sparse index for `keyword_search` tool           |
+| `PINECONE_RERANK_MODEL`            | No       | `bge-reranker-v2-m3`   | Reranking model                                  |
+| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No       | `INFO`                 | Logging level                                    |
 
 ### Claude Desktop Configuration
 
@@ -143,11 +144,12 @@ node node_modules/@will-cppa/pinecone-read-only-mcp/dist/index.js --api-key YOUR
 ### Available Options
 
 ```
---api-key TEXT        Pinecone API key (or set PINECONE_API_KEY env var)
---index-name TEXT     Pinecone index name [default: rag-hybrid]
---rerank-model TEXT   Reranking model [default: bge-reranker-v2-m3]
---log-level TEXT      Logging level [default: INFO]
---help, -h            Show help message
+--api-key TEXT           Pinecone API key (or set PINECONE_API_KEY env var)
+--index-name TEXT        Pinecone index name [default: rag-hybrid]
+--sparse-index-name TEXT Sparse index for keyword_search [default: pinecone-rag-sparse]
+--rerank-model TEXT      Reranking model [default: bge-reranker-v2-m3]
+--log-level TEXT         Logging level [default: INFO]
+--help, -h               Show help message
 ```
 
 ## Deployment
@@ -338,6 +340,45 @@ Returns the **unique document count** matching a metadata filter and semantic qu
 }
 ```
 
+### `keyword_search`
+
+Performs **keyword (lexical/sparse-only)** search over the dedicated sparse index (default: `pinecone-rag-sparse`). Use for exact or keyword-style queries. Does not use the dense index or semantic reranking. Call `list_namespaces` first to discover namespaces; `suggest_query_params` is optional.
+
+**Parameters:**
+
+| Parameter         | Type     | Required | Default | Description                                                                 |
+| ----------------- | -------- | -------- | ------- | --------------------------------------------------------------------------- |
+| `query_text`      | string   | Yes      | -       | Search query text (keyword/lexical match)                                   |
+| `namespace`       | string   | Yes      | -       | Namespace to search (use `list_namespaces` to discover)                     |
+| `top_k`           | integer  | No       | `10`    | Number of results to return (1-100)                                         |
+| `metadata_filter` | object   | No       | -       | Optional metadata filter (same operators as `query`)                         |
+| `fields`          | string[] | No       | -       | Optional field names to return; omit for all fields                        |
+
+**Returns:** JSON with `status`, `query`, `namespace`, `index` (sparse index name), `result_count`, and `results` (ids, metadata, scores). Result rows match the `query` tool shape (e.g. `paper_number`, `title`, `author`, `url`, `content`, `score`, `reranked: false`).
+
+**Example response:**
+
+```json
+{
+  "status": "success",
+  "query": "contracts C++",
+  "namespace": "wg21-papers",
+  "index": "pinecone-rag-sparse",
+  "result_count": 5,
+  "results": [
+    {
+      "paper_number": "P0548",
+      "title": "Contracts for C++",
+      "author": "John Doe",
+      "url": "https://...",
+      "content": "...",
+      "score": 0.85,
+      "reranked": false
+    }
+  ]
+}
+```
+
 ### `query`
 
 Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. For **count** questions, use the `count` tool instead.
@@ -470,6 +511,18 @@ npm run build
 npm test
 ```
 
+### Testing the keyword_search tool
+
+1. **Connectivity and keyword search (script):**  
+   Run the search test script (includes a keyword search step against the sparse index):
+   ```bash
+   PINECONE_API_KEY=your-key npm run test:search
+   ```
+   If the sparse index (`pinecone-rag-sparse` by default) does not exist or has no data, the keyword search step is skipped with a warning.
+
+2. **Via MCP client:**  
+   Start the server and call the `keyword_search` tool with `query_text`, `namespace` (from `list_namespaces`), and optional `top_k` or `metadata_filter`. Response shape is the same as the `query` tool (e.g. `results` with ids, metadata, scores; `reranked` is always `false`).
+
 ### Code Quality
 
 ```bash

scripts/test-search.ts

@@ -152,6 +152,28 @@ async function test() {
       );
     }
 
+    // Test 5: Keyword (sparse-only) search on pinecone-rag-sparse
+    console.log(`\n🔤 Test 5: Keyword search (sparse-only index)`);
+    console.log(`   Namespace: "${testNamespace}"`);
+    console.log(`   Query: "test query"`);
+    console.log(`   Top K: 3`);
+    try {
+      const startTime5 = Date.now();
+      const results5 = await client.keywordSearch({
+        query: 'test query',
+        namespace: testNamespace,
+        topK: 3,
+      });
+      const duration5 = Date.now() - startTime5;
+      console.log(`✅ Keyword search returned ${results5.length} result(s) in ${duration5}ms`);
+      if (results5.length > 0) {
+        console.log(`   First result score: ${results5[0].score.toFixed(4)}, reranked: ${results5[0].reranked}`);
+      }
+    } catch (kwError) {
+      console.log(`⚠️  Keyword search skipped: ${kwError instanceof Error ? kwError.message : String(kwError)}`);
+      console.log(`   Ensure PINECONE_SPARSE_INDEX_NAME (e.g. pinecone-rag-sparse) exists and has data.`);
+    }
+
     console.log('\n✨ All tests completed successfully!');
     console.log(`\nPerformance comparison:`);
     console.log(`  Without reranking:    ${duration1}ms`);

src/constants.ts

@@ -3,6 +3,8 @@
  */
 
 export const DEFAULT_INDEX_NAME = 'rag-hybrid';
+/** Default index name for keyword (sparse-only) search. Used by the keyword_search tool. */
+export const DEFAULT_SPARSE_INDEX_NAME = 'pinecone-rag-sparse';
 export const DEFAULT_RERANK_MODEL = 'bge-reranker-v2-m3';
 export const DEFAULT_TOP_K = 10;
 export const MAX_TOP_K = 100;
@@ -45,6 +47,7 @@ Features:
 - Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count.
 - URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url.
 - Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization.
+- Keyword search: Use keyword_search to query the sparse index (e.g. pinecone-rag-sparse) for lexical/keyword-only retrieval without reranking.
 
 Usage:
 1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index

src/index.ts

@@ -11,7 +11,7 @@
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { PineconeClient } from './pinecone-client.js';
 import { setupServer, setPineconeClient } from './server.js';
-import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL } from './constants.js';
+import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL, DEFAULT_SPARSE_INDEX_NAME } from './constants.js';
 import type { LogLevel } from './config.js';
 import { setLogLevel } from './logger.js';
 import * as dotenv from 'dotenv';
@@ -22,6 +22,7 @@ dotenv.config();
 interface CLIOptions {
   apiKey?: string;
   indexName?: string;
+  sparseIndexName?: string;
   rerankModel?: string;
   logLevel?: string;
 }
@@ -44,6 +45,10 @@ function parseArgs(): CLIOptions {
         options.indexName = nextArg;
         i++;
         break;
+      case '--sparse-index-name':
+        options.sparseIndexName = nextArg;
+        i++;
+        break;
       case '--rerank-model':
         options.rerankModel = nextArg;
         i++;
@@ -71,15 +76,17 @@ Pinecone Read-Only MCP Server
 Usage: pinecone-read-only-mcp [options]
 
 Options:
-  --api-key TEXT        Pinecone API key (or set PINECONE_API_KEY env var)
-  --index-name TEXT     Pinecone index name [default: ${DEFAULT_INDEX_NAME}]
-  --rerank-model TEXT   Reranking model [default: ${DEFAULT_RERANK_MODEL}]
-  --log-level TEXT      Logging level [default: INFO]
-  --help, -h            Show this help message
+  --api-key TEXT           Pinecone API key (or set PINECONE_API_KEY env var)
+  --index-name TEXT        Pinecone index name [default: ${DEFAULT_INDEX_NAME}]
+  --sparse-index-name TEXT Sparse index for keyword_search [default: ${DEFAULT_SPARSE_INDEX_NAME}]
+  --rerank-model TEXT      Reranking model [default: ${DEFAULT_RERANK_MODEL}]
+  --log-level TEXT         Logging level [default: INFO]
+  --help, -h               Show this help message
 
 Environment Variables:
   PINECONE_API_KEY              Pinecone API key
   PINECONE_INDEX_NAME           Pinecone index name
+  PINECONE_SPARSE_INDEX_NAME    Sparse index for keyword_search tool
   PINECONE_RERANK_MODEL         Reranking model name
   PINECONE_READ_ONLY_MCP_LOG_LEVEL  Logging level
 
@@ -119,19 +126,24 @@ async function main(): Promise<void> {
 
     // Get configuration
     const indexName = options.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME;
+    const sparseIndexName =
+      options.sparseIndexName ||
+      process.env['PINECONE_SPARSE_INDEX_NAME'] ||
+      DEFAULT_SPARSE_INDEX_NAME;
     const rerankModel =
       options.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL;
 
     // Initialize Pinecone client
     const client = new PineconeClient({
       apiKey,
       indexName,
+      sparseIndexName,
       rerankModel,
     });
     setPineconeClient(client);
 
     console.error(`Starting Pinecone Read-Only MCP server with stdio transport`);
-    console.error(`Using Pinecone index: ${indexName}`);
+    console.error(`Using Pinecone index: ${indexName} (keyword search: ${sparseIndexName})`);
     console.error(`Log level: ${logLevel}`);
 
     // Setup server

src/pinecone-client.ts

@@ -20,6 +20,7 @@ import type {
   QueryParams,
   CountParams,
   CountResult,
+  KeywordSearchParams,
   MergedHit,
   NamespaceHandle,
   SearchableIndex,
@@ -28,6 +29,7 @@ import type {
 import {
   DEFAULT_INDEX_NAME,
   DEFAULT_RERANK_MODEL,
+  DEFAULT_SPARSE_INDEX_NAME,
   DEFAULT_TOP_K,
   MAX_TOP_K,
   COUNT_TOP_K,
@@ -55,19 +57,25 @@ function inferMetadataFieldType(value: unknown): string {
 export class PineconeClient {
   private apiKey: string;
   private indexName: string;
+  private sparseIndexName: string;
   private rerankModel: string;
   private defaultTopK: number;
 
   // Lazy initialization
   private pc: Pinecone | null = null;
   private denseIndex: SearchableIndex | null = null;
   private sparseIndex: SearchableIndex | null = null;
+  private keywordIndex: SearchableIndex | null = null;
   private initialized = false;
 
   /** Create a client with the given config; env vars override index name, rerank model, and top-k. */
   constructor(config: PineconeClientConfig) {
     this.apiKey = config.apiKey;
     this.indexName = config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME;
+    this.sparseIndexName =
+      config.sparseIndexName ||
+      process.env['PINECONE_SPARSE_INDEX_NAME'] ||
+      DEFAULT_SPARSE_INDEX_NAME;
     this.rerankModel =
       config.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL;
     this.defaultTopK =
@@ -115,6 +123,21 @@ export class PineconeClient {
     return { denseIndex: dense, sparseIndex: sparse };
   }
 
+  /**
+   * Ensure the dedicated keyword (sparse-only) index is initialized.
+   * Used by the keyword_search tool to query pinecone-rag-sparse (or configured name).
+   */
+  private async ensureKeywordIndex(): Promise<SearchableIndex> {
+    if (this.keywordIndex !== null) {
+      return this.keywordIndex;
+    }
+    const pc = this.ensureClient();
+    const index = pc.index(this.sparseIndexName) as unknown as SearchableIndex;
+    this.keywordIndex = index;
+    logInfo(`Keyword search index: ${this.sparseIndexName}`);
+    return index;
+  }
+
   /**
    * List all available namespaces with their metadata information
    *
@@ -453,6 +476,68 @@ export class PineconeClient {
     return documents;
   }
 
+  /**
+   * Keyword (sparse-only) search against the dedicated sparse index.
+   * Performs lexical/keyword retrieval only—no dense index, no reranking.
+   * Use for exact or keyword-style queries on the pinecone-rag-sparse index.
+   */
+  async keywordSearch(params: KeywordSearchParams): Promise<SearchResult[]> {
+    const {
+      query,
+      namespace,
+      topK: requestedTopK,
+      metadataFilter,
+      fields: requestedFields,
+    } = params;
+
+    if (!query || !query.trim()) {
+      throw new Error('Query cannot be empty');
+    }
+
+    let topK = requestedTopK !== undefined ? requestedTopK : this.defaultTopK;
+    if (topK < 1) {
+      throw new Error('topK must be at least 1');
+    }
+    if (topK > MAX_TOP_K) {
+      topK = MAX_TOP_K;
+    }
+
+    const keywordIndex = await this.ensureKeywordIndex();
+    const searchOptions = requestedFields?.length ? { fields: requestedFields } : undefined;
+
+    const hits = await this.searchIndex(
+      keywordIndex,
+      query.trim(),
+      topK,
+      namespace,
+      metadataFilter,
+      searchOptions
+    );
+
+    const documents: SearchResult[] = hits.map((hit) => {
+      const fields = hit.fields || {};
+      let content = '';
+      const metadata: Record<string, PineconeMetadataValue> = {};
+      for (const [key, value] of Object.entries(fields)) {
+        if (key === 'chunk_text') {
+          content = typeof value === 'string' ? value : '';
+        } else {
+          metadata[key] = value as PineconeMetadataValue;
+        }
+      }
+      return {
+        id: hit._id || '',
+        content,
+        score: hit._score || 0,
+        metadata,
+        reranked: false,
+      };
+    });
+
+    logInfo(`Keyword search returned ${documents.length} results from ${this.sparseIndexName}`);
+    return documents;
+  }
+
   /**
    * Return the number of unique documents matching the query and optional metadata filter.
    * Uses semantic search only (dense index), requests minimal fields (document_number, url, doc_id)

src/server.ts

@@ -10,6 +10,7 @@ import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js
 import { registerCountTool } from './server/tools/count-tool.js';
 import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js';
 import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js';
+import { registerKeywordSearchTool } from './server/tools/keyword-search-tool.js';
 import { registerListNamespacesTool } from './server/tools/list-namespaces-tool.js';
 import { registerNamespaceRouterTool } from './server/tools/namespace-router-tool.js';
 import { registerQueryDocumentsTool } from './server/tools/query-documents-tool.js';
@@ -38,6 +39,7 @@ export async function setupServer(): Promise<McpServer> {
   registerSuggestQueryParamsTool(server);
   registerCountTool(server);
   registerQueryTool(server);
+  registerKeywordSearchTool(server);
   registerQueryDocumentsTool(server);
   registerGuidedQueryTool(server);
   registerGenerateUrlsTool(server);