Fix stale benchmark imports and add benchmarks typecheck to CI

Author: zho

.github/workflows/ci.yml

@@ -33,6 +33,9 @@ jobs:
       - name: Type check
         run: npm run typecheck
 
+      - name: Type check benchmarks
+        run: npm run typecheck:benchmarks
+
       - name: Lint
         run: npm run lint
 
@@ -45,6 +48,9 @@ jobs:
       - name: Smoke test CLI
         run: node dist/index.js --help
 
+      - name: Benchmark smoke run
+        run: npm run benchmark:smoke
+
       - name: Run tests with coverage
         run: npm run test:coverage
 

benchmarks/latency.ts

@@ -3,23 +3,25 @@
  * Local benchmark harness: mocked Pinecone I/O, measures server-side latency (p50/p95/p99).
  *
  * Usage: npm run benchmark
+ *        npm run benchmark:smoke   (fast offline run for CI)
  */
 
 import { writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { performance } from 'node:perf_hooks';
 import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
-import { PineconeClient } from '../src/pinecone-client.js';
+import { PineconeClient } from '../src/core/pinecone-client.js';
 import { setLogLevel } from '../src/logger.js';
 import { exitOnDemoFailure } from '../examples/lib/exit-on-failure.js';
-import { setPineconeClient } from '../src/server/client-context.js';
-import { invalidateNamespacesCache, getNamespacesWithCache } from '../src/server/namespaces-cache.js';
-import { registerGuidedQueryTool } from '../src/server/tools/guided-query-tool.js';
+import { setPineconeClient } from '../src/core/server/client-context.js';
+import { invalidateNamespacesCache, getNamespacesWithCache } from '../src/core/server/namespaces-cache.js';
+import { registerGuidedQueryTool } from '../src/core/server/tools/guided-query-tool.js';
 import type { MergedHit, PineconeHit, SearchResult, SearchableIndex } from '../src/types.js';
 
-const WARMUP = 10;
-const ITERATIONS = 200;
+const SMOKE = process.argv.includes('--smoke');
+const WARMUP = SMOKE ? 1 : 10;
+const ITERATIONS = SMOKE ? 3 : 200;
 const TOP_K = 20;
 
 type BenchmarkResult = {
@@ -33,17 +35,18 @@ type BenchmarkResult = {
 };
 
 /** Test double: stub ensureIndexes, searchIndex, rerankResults (no network). */
-type PineconeClientBenchDouble = PineconeClient & {
+type QueryBenchClient = {
   ensureIndexes: () => Promise<{ denseIndex: SearchableIndex; sparseIndex: SearchableIndex }>;
   searchIndex: (
-    _index: SearchableIndex,
+    index: SearchableIndex,
     _query: string,
     _topK: number,
     _namespace?: string,
     _metadataFilter?: Record<string, unknown>,
     _options?: { fields?: string[] }
   ) => Promise<PineconeHit[]>;
   rerankResults: (_q: string, results: MergedHit[], topN: number) => Promise<SearchResult[]>;
+  query: PineconeClient['query'];
 };
 
 function syntheticHits(prefix: string, count: number, scoreBase: number): PineconeHit[] {
@@ -107,7 +110,7 @@ function formatTable(rows: BenchmarkResult[]): string {
   const headers = ['Scenario', 'p50 (ms)', 'p95 (ms)', 'p99 (ms)', 'min (ms)', 'max (ms)'];
   const colWidths = [28, 12, 12, 12, 12, 12];
   const line = (cells: string[]) =>
-    cells.map((c, i) => c.padEnd(colWidths[i])).join(' | ');
+    cells.map((c, i) => c.padEnd(colWidths[i] ?? 0)).join(' | ');
   const out: string[] = [line(headers), line(colWidths.map((w) => '-'.repeat(w)))];
   for (const r of rows) {
     out.push(
@@ -124,7 +127,7 @@ function formatTable(rows: BenchmarkResult[]): string {
   return out.join('\n');
 }
 
-function buildQueryBenchClient(): PineconeClientBenchDouble {
+function buildQueryBenchClient(): QueryBenchClient {
   const denseHits = syntheticHits('dense', TOP_K, 0.95);
   const sparseHits = syntheticHits('sparse', TOP_K, 0.9);
   const denseIndexRef = {} as SearchableIndex;
@@ -133,20 +136,21 @@ function buildQueryBenchClient(): PineconeClientBenchDouble {
     apiKey: 'bench-key',
     indexName: 'bench-index',
     rerankModel: 'bench-rerank',
-  }) as PineconeClientBenchDouble;
+  });
+  const bench = client as unknown as QueryBenchClient;
 
-  client.ensureIndexes = async () => ({
+  bench.ensureIndexes = async () => ({
     denseIndex: denseIndexRef,
     sparseIndex: sparseIndexRef,
   });
 
-  client.searchIndex = async (index) => {
+  bench.searchIndex = async (index) => {
     if (index === denseIndexRef) return denseHits;
     if (index === sparseIndexRef) return sparseHits;
     return [];
   };
 
-  client.rerankResults = async (_q, results, topN) =>
+  bench.rerankResults = async (_q, results, topN) =>
     results.slice(0, topN).map((r, i) => ({
       id: r._id,
       content: r.chunk_text,
@@ -155,7 +159,7 @@ function buildQueryBenchClient(): PineconeClientBenchDouble {
       reranked: true,
     }));
 
-  return client;
+  return bench;
 }
 
 function captureGuidedQueryHandler(): (params: {
@@ -213,10 +217,10 @@ function createBenchPineconeMock(): PineconeClient {
     content: String(h.fields['chunk_text'] ?? ''),
     score: h._score,
     metadata: {
-      document_number: h.fields['document_number'],
-      title: h.fields['title'],
-      url: h.fields['url'],
-      author: h.fields['author'],
+      document_number: String(h.fields['document_number'] ?? ''),
+      title: String(h.fields['title'] ?? ''),
+      url: String(h.fields['url'] ?? ''),
+      author: String(h.fields['author'] ?? ''),
     },
     reranked: false,
   }));
@@ -251,6 +255,10 @@ function createBenchPineconeMock(): PineconeClient {
 }
 
 async function main(): Promise<void> {
+  if (SMOKE) {
+    process.env['PINECONE_API_KEY'] ??= 'bench-key';
+    process.env['PINECONE_INDEX_NAME'] ??= 'bench-index';
+  }
   setLogLevel('ERROR');
   const results: BenchmarkResult[] = [];
 
@@ -266,16 +274,19 @@ async function main(): Promise<void> {
     })
   );
 
-  results.push(
-    await runBenchmark('query_with_rerank', async () => {
-      await queryClient.query({
-        query: 'benchmark hybrid query text',
-        namespace: 'docs',
-        topK: TOP_K,
-        useReranking: true,
-      });
-    })
-  );
+  // Skipped in smoke: rerank double is dead post-restructure; would hit real Pinecone HTTP.
+  if (!SMOKE) {
+    results.push(
+      await runBenchmark('query_with_rerank', async () => {
+        await queryClient.query({
+          query: 'benchmark hybrid query text',
+          namespace: 'docs',
+          topK: TOP_K,
+          useReranking: true,
+        });
+      })
+    );
+  }
 
   setPineconeClient(createBenchPineconeMock());
   invalidateNamespacesCache();
@@ -320,11 +331,13 @@ async function main(): Promise<void> {
     results,
   };
 
-  const __filename = fileURLToPath(import.meta.url);
-  const __dirname = dirname(__filename);
-  const baselinePath = join(__dirname, 'baseline.json');
-  writeFileSync(baselinePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
-  console.log(`Wrote ${baselinePath}`);
+  if (!SMOKE) {
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    const baselinePath = join(__dirname, 'baseline.json');
+    writeFileSync(baselinePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8');
+    console.log(`Wrote ${baselinePath}`);
+  }
 }
 
 main().catch(exitOnDemoFailure('latency benchmark'));

benchmarks/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "noEmit": true,
+    "rootDir": ".."
+  },
+  "include": ["./**/*.ts", "../src/**/*.ts"],
+  "exclude": ["../node_modules", "../dist", "../**/*.test.ts", "../src/server/tools/test-helpers.ts"]
+}

package.json

@@ -59,14 +59,16 @@
     "test:coverage": "vitest run --coverage",
     "test:watch": "vitest",
     "benchmark": "tsx benchmarks/latency.ts",
+    "benchmark:smoke": "tsx benchmarks/latency.ts --smoke",
     "test:search": "tsx scripts/test-search.ts",
     "lint": "eslint src/",
     "lint:fix": "eslint src/ --fix",
     "format": "prettier --write \"src/**/*.ts\" \"*.json\" \".prettierrc\"",
     "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"",
     "docs:link-check": "node scripts/docs-link-check.mjs",
     "typecheck": "tsc --noEmit",
-    "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm run test:coverage",
+    "typecheck:benchmarks": "tsc -p benchmarks/tsconfig.json",
+    "ci": "npm run typecheck && npm run typecheck:benchmarks && npm run lint && npm run format:check && npm run build && npm run benchmark:smoke && npm run test:coverage",
     "release:check": "npm run ci && npm pack --dry-run --ignore-scripts",
     "ci:local": "bash scripts/ci-local.sh",
     "prepublishOnly": "npm run ci",