Merge pull request #31 from CppDigest/chen/issue-29-keyword-search

wpak-ai · web-flow · commit 9b74d6c4cfc1 · 2026-03-09T13:43:01.000-04:00
feat: add keyword_search tool for pinecone-rag-sparse index (#29)
diff --git a/README.md b/README.md
@@ -58,12 +58,12 @@ The server requires a Pinecone API key and supports the following configuration
 
 ### Environment Variables
 
-| Variable                           | Required | Default              | Description           |
-| ---------------------------------- | -------- | -------------------- | --------------------- |
-| `PINECONE_API_KEY`                 | Yes      | -                    | Your Pinecone API key |
-| `PINECONE_INDEX_NAME`              | No       | `rag-hybrid`         | Pinecone index name   |
-| `PINECONE_RERANK_MODEL`            | No       | `bge-reranker-v2-m3` | Reranking model       |
-| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No       | `INFO`               | Logging level         |
+| Variable                           | Required | Default                | Description                                      |
+| ---------------------------------- | -------- | ---------------------- | ------------------------------------------------ |
+| `PINECONE_API_KEY`                 | Yes      | -                      | Your Pinecone API key                            |
+| `PINECONE_INDEX_NAME`              | No       | `rag-hybrid`           | Pinecone index name (dense + sparse for hybrid)   |
+| `PINECONE_RERANK_MODEL`            | No       | `bge-reranker-v2-m3`   | Reranking model                                  |
+| `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No       | `INFO`                 | Logging level                                    |
 
 ### Claude Desktop Configuration
 
@@ -143,11 +143,11 @@ node node_modules/@will-cppa/pinecone-read-only-mcp/dist/index.js --api-key YOUR
 ### Available Options
 
 ```
---api-key TEXT        Pinecone API key (or set PINECONE_API_KEY env var)
---index-name TEXT     Pinecone index name [default: rag-hybrid]
---rerank-model TEXT   Reranking model [default: bge-reranker-v2-m3]
---log-level TEXT      Logging level [default: INFO]
---help, -h            Show help message
+--api-key TEXT           Pinecone API key (or set PINECONE_API_KEY env var)
+--index-name TEXT        Pinecone index name [default: rag-hybrid]
+--rerank-model TEXT      Reranking model [default: bge-reranker-v2-m3]
+--log-level TEXT         Logging level [default: INFO]
+--help, -h               Show help message
 ```
 
 ## Deployment
@@ -338,6 +338,45 @@ Returns the **unique document count** matching a metadata filter and semantic qu
 }
 ```
 
+### `keyword_search`
+
+Performs **keyword (lexical/sparse-only)** search over the dedicated sparse index (default: `rag-hybrid-sparse`, i.e. `{PINECONE_INDEX_NAME}-sparse`). Use for exact or keyword-style queries. Does not use the dense index or semantic reranking. Call `list_namespaces` first to discover namespaces; `suggest_query_params` is optional.
+
+**Parameters:**
+
+| Parameter         | Type     | Required | Default | Description                                                                 |
+| ----------------- | -------- | -------- | ------- | --------------------------------------------------------------------------- |
+| `query_text`      | string   | Yes      | -       | Search query text (keyword/lexical match)                                   |
+| `namespace`       | string   | Yes      | -       | Namespace to search (use `list_namespaces` to discover)                     |
+| `top_k`           | integer  | No       | `10`    | Number of results to return (1-100)                                         |
+| `metadata_filter` | object   | No       | -       | Optional metadata filter (same operators as `query`)                         |
+| `fields`          | string[] | No       | -       | Optional field names to return; omit for all fields                        |
+
+**Returns:** JSON with `status`, `query`, `namespace`, `index` (sparse index name), `result_count`, and `results` (ids, metadata, scores). Result rows match the `query` tool shape (e.g. `paper_number`, `title`, `author`, `url`, `content`, `score`, `reranked: false`).
+
+**Example response:**
+
+```json
+{
+  "status": "success",
+  "query": "contracts C++",
+  "namespace": "wg21-papers",
+  "index": "rag-hybrid-sparse",
+  "result_count": 5,
+  "results": [
+    {
+      "paper_number": "P0548",
+      "title": "Contracts for C++",
+      "author": "John Doe",
+      "url": "https://...",
+      "content": "...",
+      "score": 0.85,
+      "reranked": false
+    }
+  ]
+}
+```
+
 ### `query`
 
 Performs hybrid semantic search over the specified namespace in the Pinecone index with optional metadata filtering. For **count** questions, use the `count` tool instead.
@@ -470,6 +509,18 @@ npm run build
 npm test
 ```
 
+### Testing the keyword_search tool
+
+1. **Connectivity and keyword search (script):**  
+   Run the search test script (includes a keyword search step against the sparse index):
+   ```bash
+   PINECONE_API_KEY=your-key npm run test:search
+   ```
+   If the sparse index (`rag-hybrid-sparse` by default) does not exist or has no data, the keyword search step is skipped with a warning.
+
+2. **Via MCP client:**  
+   Start the server and call the `keyword_search` tool with `query_text`, `namespace` (from `list_namespaces`), and optional `top_k` or `metadata_filter`. Response shape is the same as the `query` tool (e.g. `results` with ids, metadata, scores; `reranked` is always `false`).
+
 ### Code Quality
 
 ```bash
diff --git a/scripts/test-search.ts b/scripts/test-search.ts
@@ -152,13 +152,62 @@ async function test() {
       );
     }
 
+    // Test 5: Keyword (sparse-only) search — use namespace from sparse index, not dense
+    let duration5: number | undefined;
+    let test5Skipped = false;
+    const sparseNamespaces = await client.listNamespacesFromKeywordIndex();
+    if (sparseNamespaces.length === 0) {
+      test5Skipped = true;
+      console.log(`\n🔤 Test 5: Keyword search (sparse-only index)`);
+      console.log(
+        `⚠️  Keyword search skipped: sparse index has no namespaces (or index unavailable).`
+      );
+      console.log(
+        `   Ensure the sparse index (e.g. rag-hybrid-sparse) exists and has data.`
+      );
+    } else {
+      const sparseTestNamespace = sparseNamespaces[0].namespace;
+      console.log(`\n🔤 Test 5: Keyword search (sparse-only index)`);
+      console.log(`   Namespace: "${sparseTestNamespace}" (from sparse index)`);
+      console.log(`   Query: "test query"`);
+      console.log(`   Top K: 3`);
+      try {
+        const startTime5 = Date.now();
+        const results5 = await client.keywordSearch({
+          query: 'test query',
+          namespace: sparseTestNamespace,
+          topK: 3,
+        });
+        duration5 = Date.now() - startTime5;
+        console.log(`✅ Keyword search returned ${results5.length} result(s) in ${duration5}ms`);
+        if (results5.length > 0) {
+          console.log(
+            `   First result score: ${results5[0].score.toFixed(4)}, reranked: ${results5[0].reranked}`
+          );
+        }
+      } catch (kwError) {
+        test5Skipped = true;
+        console.log(
+          `⚠️  Keyword search skipped: ${kwError instanceof Error ? kwError.message : String(kwError)}`
+        );
+        console.log(
+          `   Ensure the sparse index exists and namespace "${sparseTestNamespace}" has data.`
+        );
+      }
+    }
+
     console.log('\n✨ All tests completed successfully!');
     console.log(`\nPerformance comparison:`);
     console.log(`  Without reranking:    ${duration1}ms`);
     console.log(`  With reranking:       ${duration2}ms`);
     if (duration3 !== undefined) {
       console.log(`  With metadata filter: ${duration3}ms`);
     }
+    if (duration5 !== undefined) {
+      console.log(`  Keyword search:       ${duration5}ms`);
+    } else if (test5Skipped) {
+      console.log(`  Keyword search:       skipped`);
+    }
     console.log(`  Reranking overhead:   ${duration2 - duration1}ms`);
   } catch (error) {
     console.error('\n❌ Error during testing:', error);
diff --git a/src/constants.ts b/src/constants.ts
@@ -45,6 +45,7 @@ Features:
 - Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count.
 - URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url.
 - Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization.
+- Keyword search: Use keyword_search to query the sparse index (default: rag-hybrid-sparse) for lexical/keyword-only retrieval without reranking.
 
 Usage:
 1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index
diff --git a/src/index.ts b/src/index.ts
@@ -71,15 +71,15 @@ Pinecone Read-Only MCP Server
 Usage: pinecone-read-only-mcp [options]
 
 Options:
-  --api-key TEXT        Pinecone API key (or set PINECONE_API_KEY env var)
-  --index-name TEXT     Pinecone index name [default: ${DEFAULT_INDEX_NAME}]
-  --rerank-model TEXT   Reranking model [default: ${DEFAULT_RERANK_MODEL}]
-  --log-level TEXT      Logging level [default: INFO]
-  --help, -h            Show this help message
+  --api-key TEXT           Pinecone API key (or set PINECONE_API_KEY env var)
+  --index-name TEXT        Pinecone index name [default: ${DEFAULT_INDEX_NAME}]; sparse index is {index-name}-sparse
+  --rerank-model TEXT      Reranking model [default: ${DEFAULT_RERANK_MODEL}]
+  --log-level TEXT         Logging level [default: INFO]
+  --help, -h               Show this help message
 
 Environment Variables:
   PINECONE_API_KEY              Pinecone API key
-  PINECONE_INDEX_NAME           Pinecone index name
+  PINECONE_INDEX_NAME           Pinecone index name (sparse index: {name}-sparse)
   PINECONE_RERANK_MODEL         Reranking model name
   PINECONE_READ_ONLY_MCP_LOG_LEVEL  Logging level
 
@@ -131,7 +131,7 @@ async function main(): Promise<void> {
     setPineconeClient(client);
 
     console.error(`Starting Pinecone Read-Only MCP server with stdio transport`);
-    console.error(`Using Pinecone index: ${indexName}`);
+    console.error(`Using Pinecone index: ${indexName} (sparse: ${indexName}-sparse)`);
     console.error(`Log level: ${logLevel}`);
 
     // Setup server
diff --git a/src/pinecone-client.ts b/src/pinecone-client.ts
@@ -20,6 +20,7 @@ import type {
   QueryParams,
   CountParams,
   CountResult,
+  KeywordSearchParams,
   MergedHit,
   NamespaceHandle,
   SearchableIndex,
@@ -74,6 +75,28 @@ export class PineconeClient {
       config.defaultTopK || parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K));
   }
 
+  /** Returns the sparse index name (same as hybrid sparse: {indexName}-sparse). Used for keyword_search response. */
+  getSparseIndexName(): string {
+    return `${this.indexName}-sparse`;
+  }
+
+  /**
+   * Normalize and clamp topK from request (validates >= 1, caps at MAX_TOP_K).
+   */
+  private clampTopK(requested: number | undefined): number {
+    if (requested !== undefined && !Number.isFinite(requested)) {
+      throw new Error('topK must be a finite number >= 1');
+    }
+    let topK = requested !== undefined ? requested : this.defaultTopK;
+    if (topK < 1) {
+      throw new Error('topK must be at least 1');
+    }
+    if (topK > MAX_TOP_K) {
+      topK = MAX_TOP_K;
+    }
+    return topK;
+  }
+
   /**
    * Ensure Pinecone client is initialized
    */
@@ -103,7 +126,7 @@ export class PineconeClient {
 
     const pc = this.ensureClient();
     const denseName = this.indexName;
-    const sparseName = `${this.indexName}-sparse`;
+    const sparseName = this.getSparseIndexName();
 
     const dense = pc.index(denseName) as unknown as SearchableIndex;
     const sparse = pc.index(sparseName) as unknown as SearchableIndex;
@@ -115,6 +138,29 @@ export class PineconeClient {
     return { denseIndex: dense, sparseIndex: sparse };
   }
 
+  /**
+   * List namespaces present on the sparse index (same index used for hybrid sparse and keyword_search).
+   * Use this to choose a namespace for sparse-only queries instead of the dense index list.
+   */
+  async listNamespacesFromKeywordIndex(): Promise<
+    Array<{ namespace: string; recordCount: number }>
+  > {
+    try {
+      const { sparseIndex } = await this.ensureIndexes();
+      const stats = sparseIndex.describeIndexStats
+        ? await sparseIndex.describeIndexStats()
+        : undefined;
+      const namespaces = stats?.namespaces ?? {};
+      return Object.entries(namespaces).map(([namespace, info]) => ({
+        namespace,
+        recordCount: info?.recordCount ?? 0,
+      }));
+    } catch (error) {
+      logError('Error listing namespaces from keyword index', error);
+      return [];
+    }
+  }
+
   /**
    * List all available namespaces with their metadata information
    *
@@ -390,14 +436,7 @@ export class PineconeClient {
       throw new Error('Query cannot be empty');
     }
 
-    let topK = requestedTopK !== undefined ? requestedTopK : this.defaultTopK;
-    if (topK < 1) {
-      throw new Error('topK must be at least 1');
-    }
-
-    if (topK > MAX_TOP_K) {
-      topK = MAX_TOP_K;
-    }
+    const topK = this.clampTopK(requestedTopK);
 
     // When reranking, Pinecone requires chunk_text in returned fields; add it if user specified fields without it
     const searchFields =
@@ -453,6 +492,64 @@ export class PineconeClient {
     return documents;
   }
 
+  /**
+   * Keyword (sparse-only) search against the dedicated sparse index.
+   * Performs lexical/keyword retrieval only—no dense index, no reranking.
+   * Use for exact or keyword-style queries on the configured sparse index.
+   */
+  async keywordSearch(params: KeywordSearchParams): Promise<SearchResult[]> {
+    const {
+      query,
+      namespace,
+      topK: requestedTopK,
+      metadataFilter,
+      fields: requestedFields,
+    } = params;
+
+    if (!query || !query.trim()) {
+      throw new Error('Query cannot be empty');
+    }
+
+    const topK = this.clampTopK(requestedTopK);
+
+    const { sparseIndex } = await this.ensureIndexes();
+    const searchOptions = requestedFields?.length ? { fields: requestedFields } : undefined;
+
+    const hits = await this.searchIndex(
+      sparseIndex,
+      query.trim(),
+      topK,
+      namespace,
+      metadataFilter,
+      searchOptions
+    );
+
+    const documents: SearchResult[] = hits.map((hit) => {
+      const fields = hit.fields || {};
+      let content = '';
+      const metadata: Record<string, PineconeMetadataValue> = {};
+      for (const [key, value] of Object.entries(fields)) {
+        if (key === 'chunk_text') {
+          content = typeof value === 'string' ? value : '';
+        } else {
+          metadata[key] = value as PineconeMetadataValue;
+        }
+      }
+      return {
+        id: hit._id || '',
+        content,
+        score: hit._score || 0,
+        metadata,
+        reranked: false,
+      };
+    });
+
+    logInfo(
+      `Keyword search returned ${documents.length} results from ${this.getSparseIndexName()}`
+    );
+    return documents;
+  }
+
   /**
    * Return the number of unique documents matching the query and optional metadata filter.
    * Uses semantic search only (dense index), requests minimal fields (document_number, url, doc_id)
diff --git a/src/server.ts b/src/server.ts
@@ -10,6 +10,7 @@ import { SERVER_INSTRUCTIONS, SERVER_NAME, SERVER_VERSION } from './constants.js
 import { registerCountTool } from './server/tools/count-tool.js';
 import { registerGuidedQueryTool } from './server/tools/guided-query-tool.js';
 import { registerGenerateUrlsTool } from './server/tools/generate-urls-tool.js';
+import { registerKeywordSearchTool } from './server/tools/keyword-search-tool.js';
 import { registerListNamespacesTool } from './server/tools/list-namespaces-tool.js';
 import { registerNamespaceRouterTool } from './server/tools/namespace-router-tool.js';
 import { registerQueryDocumentsTool } from './server/tools/query-documents-tool.js';
@@ -38,6 +39,7 @@ export async function setupServer(): Promise<McpServer> {
   registerSuggestQueryParamsTool(server);
   registerCountTool(server);
   registerQueryTool(server);
+  registerKeywordSearchTool(server);
   registerQueryDocumentsTool(server);
   registerGuidedQueryTool(server);
   registerGenerateUrlsTool(server);
diff --git a/src/server/tools/keyword-search-tool.ts b/src/server/tools/keyword-search-tool.ts
diff --git a/src/types.ts b/src/types.ts
