refactor: use single sparse index for hybrid and keyword search (reviewer feedback)

refactor: use single sparse index for hybrid and keyword search

- Remove keywordIndex; use sparseIndex from ensureIndexes() for keywordSearch and listNamespacesFromKeywordIndex
- Remove sparseIndexName config (PINECONE_SPARSE_INDEX_NAME, --sparse-index-name) and DEFAULT_SPARSE_INDEX_NAME
- getSparseIndexName() now returns ${indexName}-sparse. Addresses reviewer: sparse_index and keyword_index are the same index

Author: clean6378-max-it

README.md

@@ -62,7 +62,6 @@ The server requires a Pinecone API key and supports the following configuration
 | ---------------------------------- | -------- | ---------------------- | ------------------------------------------------ |
 | `PINECONE_API_KEY`                 | Yes      | -                      | Your Pinecone API key                            |
 | `PINECONE_INDEX_NAME`              | No       | `rag-hybrid`           | Pinecone index name (dense + sparse for hybrid)   |
-| `PINECONE_SPARSE_INDEX_NAME`       | No       | `pinecone-rag-sparse`  | Sparse index for `keyword_search` tool           |
 | `PINECONE_RERANK_MODEL`            | No       | `bge-reranker-v2-m3`   | Reranking model                                  |
 | `PINECONE_READ_ONLY_MCP_LOG_LEVEL` | No       | `INFO`                 | Logging level                                    |
 
@@ -146,7 +145,6 @@ node node_modules/@will-cppa/pinecone-read-only-mcp/dist/index.js --api-key YOUR
 ```
 --api-key TEXT           Pinecone API key (or set PINECONE_API_KEY env var)
 --index-name TEXT        Pinecone index name [default: rag-hybrid]
---sparse-index-name TEXT Sparse index for keyword_search [default: pinecone-rag-sparse]
 --rerank-model TEXT      Reranking model [default: bge-reranker-v2-m3]
 --log-level TEXT         Logging level [default: INFO]
 --help, -h               Show help message
@@ -342,7 +340,7 @@ Returns the **unique document count** matching a metadata filter and semantic qu
 
 ### `keyword_search`
 
-Performs **keyword (lexical/sparse-only)** search over the dedicated sparse index (default: `pinecone-rag-sparse`). Use for exact or keyword-style queries. Does not use the dense index or semantic reranking. Call `list_namespaces` first to discover namespaces; `suggest_query_params` is optional.
+Performs **keyword (lexical/sparse-only)** search over the dedicated sparse index (default: `rag-hybrid-sparse`, i.e. `{PINECONE_INDEX_NAME}-sparse`). Use for exact or keyword-style queries. Does not use the dense index or semantic reranking. Call `list_namespaces` first to discover namespaces; `suggest_query_params` is optional.
 
 **Parameters:**
 
@@ -363,7 +361,7 @@ Performs **keyword (lexical/sparse-only)** search over the dedicated sparse inde
   "status": "success",
   "query": "contracts C++",
   "namespace": "wg21-papers",
-  "index": "pinecone-rag-sparse",
+  "index": "rag-hybrid-sparse",
   "result_count": 5,
   "results": [
     {
@@ -518,7 +516,7 @@ npm test
    ```bash
    PINECONE_API_KEY=your-key npm run test:search
    ```
-   If the sparse index (`pinecone-rag-sparse` by default) does not exist or has no data, the keyword search step is skipped with a warning.
+   If the sparse index (`rag-hybrid-sparse` by default) does not exist or has no data, the keyword search step is skipped with a warning.
 
 2. **Via MCP client:**  
    Start the server and call the `keyword_search` tool with `query_text`, `namespace` (from `list_namespaces`), and optional `top_k` or `metadata_filter`. Response shape is the same as the `query` tool (e.g. `results` with ids, metadata, scores; `reranked` is always `false`).

scripts/test-search.ts

@@ -163,7 +163,7 @@ async function test() {
         `⚠️  Keyword search skipped: sparse index has no namespaces (or index unavailable).`
       );
       console.log(
-        `   Ensure PINECONE_SPARSE_INDEX_NAME (e.g. pinecone-rag-sparse or rag-hybrid-sparse) exists and has data.`
+        `   Ensure the sparse index (e.g. rag-hybrid-sparse) exists and has data.`
       );
     } else {
       const sparseTestNamespace = sparseNamespaces[0].namespace;
@@ -191,7 +191,7 @@ async function test() {
           `⚠️  Keyword search skipped: ${kwError instanceof Error ? kwError.message : String(kwError)}`
         );
         console.log(
-          `   Ensure PINECONE_SPARSE_INDEX_NAME exists and namespace "${sparseTestNamespace}" has data.`
+          `   Ensure the sparse index exists and namespace "${sparseTestNamespace}" has data.`
         );
       }
     }

src/constants.ts

@@ -3,8 +3,6 @@
  */
 
 export const DEFAULT_INDEX_NAME = 'rag-hybrid';
-/** Default index name for keyword (sparse-only) search. Used by the keyword_search tool. */
-export const DEFAULT_SPARSE_INDEX_NAME = 'pinecone-rag-sparse';
 export const DEFAULT_RERANK_MODEL = 'bge-reranker-v2-m3';
 export const DEFAULT_TOP_K = 10;
 export const MAX_TOP_K = 100;
@@ -47,7 +45,7 @@ Features:
 - Count: Use the count tool for "how many X?" questions; it uses semantic search only and minimal fields (no content) for performance, returning unique document count.
 - URL Generation: Use generate_urls to synthesize URLs for namespaces that support it when metadata lacks url.
 - Document reassembly: Use query_documents to get whole documents (chunks grouped and merged by document_number/doc_id/url) for content analysis or summarization.
-- Keyword search: Use keyword_search to query the sparse index (e.g. pinecone-rag-sparse) for lexical/keyword-only retrieval without reranking.
+- Keyword search: Use keyword_search to query the sparse index (default: rag-hybrid-sparse) for lexical/keyword-only retrieval without reranking.
 
 Usage:
 1. Use list_namespaces (cached for 30 minutes) to discover available namespaces in the index

src/index.ts

@@ -11,11 +11,7 @@
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { PineconeClient } from './pinecone-client.js';
 import { setupServer, setPineconeClient } from './server.js';
-import {
-  DEFAULT_INDEX_NAME,
-  DEFAULT_RERANK_MODEL,
-  DEFAULT_SPARSE_INDEX_NAME,
-} from './constants.js';
+import { DEFAULT_INDEX_NAME, DEFAULT_RERANK_MODEL } from './constants.js';
 import type { LogLevel } from './config.js';
 import { setLogLevel } from './logger.js';
 import * as dotenv from 'dotenv';
@@ -26,7 +22,6 @@ dotenv.config();
 interface CLIOptions {
   apiKey?: string;
   indexName?: string;
-  sparseIndexName?: string;
   rerankModel?: string;
   logLevel?: string;
 }
@@ -49,10 +44,6 @@ function parseArgs(): CLIOptions {
         options.indexName = nextArg;
         i++;
         break;
-      case '--sparse-index-name':
-        options.sparseIndexName = nextArg;
-        i++;
-        break;
       case '--rerank-model':
         options.rerankModel = nextArg;
         i++;
@@ -81,16 +72,14 @@ Usage: pinecone-read-only-mcp [options]
 
 Options:
   --api-key TEXT           Pinecone API key (or set PINECONE_API_KEY env var)
-  --index-name TEXT        Pinecone index name [default: ${DEFAULT_INDEX_NAME}]
-  --sparse-index-name TEXT Sparse index for keyword_search [default: ${DEFAULT_SPARSE_INDEX_NAME}]
+  --index-name TEXT        Pinecone index name [default: ${DEFAULT_INDEX_NAME}]; sparse index is {index-name}-sparse
   --rerank-model TEXT      Reranking model [default: ${DEFAULT_RERANK_MODEL}]
   --log-level TEXT         Logging level [default: INFO]
   --help, -h               Show this help message
 
 Environment Variables:
   PINECONE_API_KEY              Pinecone API key
-  PINECONE_INDEX_NAME           Pinecone index name
-  PINECONE_SPARSE_INDEX_NAME    Sparse index for keyword_search tool
+  PINECONE_INDEX_NAME           Pinecone index name (sparse index: {name}-sparse)
   PINECONE_RERANK_MODEL         Reranking model name
   PINECONE_READ_ONLY_MCP_LOG_LEVEL  Logging level
 
@@ -130,24 +119,19 @@ async function main(): Promise<void> {
 
     // Get configuration
     const indexName = options.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME;
-    const sparseIndexName =
-      options.sparseIndexName ||
-      process.env['PINECONE_SPARSE_INDEX_NAME'] ||
-      DEFAULT_SPARSE_INDEX_NAME;
     const rerankModel =
       options.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL;
 
     // Initialize Pinecone client
     const client = new PineconeClient({
       apiKey,
       indexName,
-      sparseIndexName,
       rerankModel,
     });
     setPineconeClient(client);
 
     console.error(`Starting Pinecone Read-Only MCP server with stdio transport`);
-    console.error(`Using Pinecone index: ${indexName} (keyword search: ${sparseIndexName})`);
+    console.error(`Using Pinecone index: ${indexName} (sparse: ${indexName}-sparse)`);
     console.error(`Log level: ${logLevel}`);
 
     // Setup server

src/pinecone-client.ts

@@ -29,7 +29,6 @@ import type {
 import {
   DEFAULT_INDEX_NAME,
   DEFAULT_RERANK_MODEL,
-  DEFAULT_SPARSE_INDEX_NAME,
   DEFAULT_TOP_K,
   MAX_TOP_K,
   COUNT_TOP_K,
@@ -57,34 +56,28 @@ function inferMetadataFieldType(value: unknown): string {
 export class PineconeClient {
   private apiKey: string;
   private indexName: string;
-  private sparseIndexName: string;
   private rerankModel: string;
   private defaultTopK: number;
 
   // Lazy initialization
   private pc: Pinecone | null = null;
   private denseIndex: SearchableIndex | null = null;
   private sparseIndex: SearchableIndex | null = null;
-  private keywordIndex: SearchableIndex | null = null;
   private initialized = false;
 
   /** Create a client with the given config; env vars override index name, rerank model, and top-k. */
   constructor(config: PineconeClientConfig) {
     this.apiKey = config.apiKey;
     this.indexName = config.indexName || process.env['PINECONE_INDEX_NAME'] || DEFAULT_INDEX_NAME;
-    this.sparseIndexName =
-      config.sparseIndexName ||
-      process.env['PINECONE_SPARSE_INDEX_NAME'] ||
-      DEFAULT_SPARSE_INDEX_NAME;
     this.rerankModel =
       config.rerankModel || process.env['PINECONE_RERANK_MODEL'] || DEFAULT_RERANK_MODEL;
     this.defaultTopK =
       config.defaultTopK || parseInt(process.env['PINECONE_TOP_K'] || String(DEFAULT_TOP_K));
   }
 
-  /** Returns the configured sparse index name used for keyword search (runtime value). */
+  /** Returns the sparse index name (same as hybrid sparse: {indexName}-sparse). Used for keyword_search response. */
   getSparseIndexName(): string {
-    return this.sparseIndexName;
+    return `${this.indexName}-sparse`;
   }
 
   /**
@@ -143,31 +136,16 @@ export class PineconeClient {
   }
 
   /**
-   * Ensure the dedicated keyword (sparse-only) index is initialized.
-   * Used by the keyword_search tool to query pinecone-rag-sparse (or configured name).
-   */
-  private async ensureKeywordIndex(): Promise<SearchableIndex> {
-    if (this.keywordIndex !== null) {
-      return this.keywordIndex;
-    }
-    const pc = this.ensureClient();
-    const index = pc.index(this.sparseIndexName) as unknown as SearchableIndex;
-    this.keywordIndex = index;
-    logInfo(`Keyword search index: ${this.sparseIndexName}`);
-    return index;
-  }
-
-  /**
-   * List namespaces present on the keyword (sparse) index used for keyword_search.
+   * List namespaces present on the sparse index (same index used for hybrid sparse and keyword_search).
    * Use this to choose a namespace for sparse-only queries instead of the dense index list.
    */
   async listNamespacesFromKeywordIndex(): Promise<
     Array<{ namespace: string; recordCount: number }>
   > {
     try {
-      const keywordIndex = await this.ensureKeywordIndex();
-      const stats = keywordIndex.describeIndexStats
-        ? await keywordIndex.describeIndexStats()
+      const { sparseIndex } = await this.ensureIndexes();
+      const stats = sparseIndex.describeIndexStats
+        ? await sparseIndex.describeIndexStats()
         : undefined;
       const namespaces = stats?.namespaces ?? {};
       return Object.entries(namespaces).map(([namespace, info]) => ({
@@ -514,7 +492,7 @@ export class PineconeClient {
   /**
    * Keyword (sparse-only) search against the dedicated sparse index.
    * Performs lexical/keyword retrieval only—no dense index, no reranking.
-   * Use for exact or keyword-style queries on the pinecone-rag-sparse index.
+   * Use for exact or keyword-style queries on the configured sparse index.
    */
   async keywordSearch(params: KeywordSearchParams): Promise<SearchResult[]> {
     const {
@@ -531,11 +509,11 @@ export class PineconeClient {
 
     const topK = this.clampTopK(requestedTopK);
 
-    const keywordIndex = await this.ensureKeywordIndex();
+    const { sparseIndex } = await this.ensureIndexes();
     const searchOptions = requestedFields?.length ? { fields: requestedFields } : undefined;
 
     const hits = await this.searchIndex(
-      keywordIndex,
+      sparseIndex,
       query.trim(),
       topK,
       namespace,
@@ -563,7 +541,7 @@ export class PineconeClient {
       };
     });
 
-    logInfo(`Keyword search returned ${documents.length} results from ${this.sparseIndexName}`);
+    logInfo(`Keyword search returned ${documents.length} results from ${this.getSparseIndexName()}`);
     return documents;
   }
 

src/server/tools/keyword-search-tool.ts

@@ -84,7 +84,7 @@ export function registerKeywordSearchTool(server: McpServer): void {
     'keyword_search',
     {
       description:
-        'Keyword (lexical/sparse-only) search over the Pinecone sparse index (e.g. pinecone-rag-sparse). ' +
+        'Keyword (lexical/sparse-only) search over the Pinecone sparse index (default: rag-hybrid-sparse). ' +
         'Use for exact or keyword-style queries. Does not use semantic reranking. ' +
         'Call list_namespaces first to discover namespaces; suggest_query_params is optional.',
       inputSchema: {

src/types.ts

@@ -8,8 +8,6 @@ export type PineconeMetadataValue = string | number | boolean | string[];
 export interface PineconeClientConfig {
   apiKey: string;
   indexName?: string;
-  /** Dedicated sparse index for keyword_search tool (e.g. pinecone-rag-sparse). */
-  sparseIndexName?: string;
   rerankModel?: string;
   defaultTopK?: number;
 }