diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 588e318..2508173 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,9 @@ jobs: - name: Type check run: npm run typecheck + - name: Type check benchmarks + run: npm run typecheck:benchmarks + - name: Lint run: npm run lint @@ -45,6 +48,9 @@ jobs: - name: Smoke test CLI run: node dist/index.js --help + - name: Benchmark smoke run + run: npm run benchmark:smoke + - name: Run tests with coverage run: npm run test:coverage diff --git a/benchmarks/latency.ts b/benchmarks/latency.ts index ddeda7a..fd7951b 100644 --- a/benchmarks/latency.ts +++ b/benchmarks/latency.ts @@ -3,6 +3,7 @@ * Local benchmark harness: mocked Pinecone I/O, measures server-side latency (p50/p95/p99). * * Usage: npm run benchmark + * npm run benchmark:smoke (fast offline run for CI) */ import { writeFileSync } from 'node:fs'; @@ -10,16 +11,17 @@ import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { performance } from 'node:perf_hooks'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; -import { PineconeClient } from '../src/pinecone-client.js'; +import { PineconeClient } from '../src/core/pinecone-client.js'; import { setLogLevel } from '../src/logger.js'; import { exitOnDemoFailure } from '../examples/lib/exit-on-failure.js'; -import { setPineconeClient } from '../src/server/client-context.js'; -import { invalidateNamespacesCache, getNamespacesWithCache } from '../src/server/namespaces-cache.js'; -import { registerGuidedQueryTool } from '../src/server/tools/guided-query-tool.js'; +import { setPineconeClient } from '../src/core/server/client-context.js'; +import { invalidateNamespacesCache, getNamespacesWithCache } from '../src/core/server/namespaces-cache.js'; +import { registerGuidedQueryTool } from '../src/core/server/tools/guided-query-tool.js'; import type { MergedHit, PineconeHit, SearchResult, SearchableIndex } from '../src/types.js'; -const WARMUP = 10; -const ITERATIONS = 200; +const SMOKE = process.argv.includes('--smoke'); +const WARMUP = SMOKE ? 1 : 10; +const ITERATIONS = SMOKE ? 3 : 200; const TOP_K = 20; type BenchmarkResult = { @@ -32,11 +34,11 @@ type BenchmarkResult = { iterations: number; }; -/** Test double: stub ensureIndexes, searchIndex, rerankResults (no network). */ -type PineconeClientBenchDouble = PineconeClient & { +/** Test double: stub ensureIndexes and searchIndex (no network). rerankResults on the instance is ineffective post-restructure — query() reranks via rerankResultsImpl(indexSession.ensureClient(), …) instead. */ +type QueryBenchClient = { ensureIndexes: () => Promise<{ denseIndex: SearchableIndex; sparseIndex: SearchableIndex }>; searchIndex: ( - _index: SearchableIndex, + index: SearchableIndex, _query: string, _topK: number, _namespace?: string, @@ -44,6 +46,7 @@ type PineconeClientBenchDouble = PineconeClient & { _options?: { fields?: string[] } ) => Promise; rerankResults: (_q: string, results: MergedHit[], topN: number) => Promise; + query: PineconeClient['query']; }; function syntheticHits(prefix: string, count: number, scoreBase: number): PineconeHit[] { @@ -107,7 +110,7 @@ function formatTable(rows: BenchmarkResult[]): string { const headers = ['Scenario', 'p50 (ms)', 'p95 (ms)', 'p99 (ms)', 'min (ms)', 'max (ms)']; const colWidths = [28, 12, 12, 12, 12, 12]; const line = (cells: string[]) => - cells.map((c, i) => c.padEnd(colWidths[i])).join(' | '); + cells.map((c, i) => c.padEnd(colWidths[i] ?? 0)).join(' | '); const out: string[] = [line(headers), line(colWidths.map((w) => '-'.repeat(w)))]; for (const r of rows) { out.push( @@ -124,7 +127,7 @@ function formatTable(rows: BenchmarkResult[]): string { return out.join('\n'); } -function buildQueryBenchClient(): PineconeClientBenchDouble { +function buildQueryBenchClient(): QueryBenchClient { const denseHits = syntheticHits('dense', TOP_K, 0.95); const sparseHits = syntheticHits('sparse', TOP_K, 0.9); const denseIndexRef = {} as SearchableIndex; @@ -133,20 +136,21 @@ function buildQueryBenchClient(): PineconeClientBenchDouble { apiKey: 'bench-key', indexName: 'bench-index', rerankModel: 'bench-rerank', - }) as PineconeClientBenchDouble; + }); + const bench = client as unknown as QueryBenchClient; - client.ensureIndexes = async () => ({ + bench.ensureIndexes = async () => ({ denseIndex: denseIndexRef, sparseIndex: sparseIndexRef, }); - client.searchIndex = async (index) => { + bench.searchIndex = async (index) => { if (index === denseIndexRef) return denseHits; if (index === sparseIndexRef) return sparseHits; return []; }; - client.rerankResults = async (_q, results, topN) => + bench.rerankResults = async (_q, results, topN) => results.slice(0, topN).map((r, i) => ({ id: r._id, content: r.chunk_text, @@ -155,7 +159,7 @@ function buildQueryBenchClient(): PineconeClientBenchDouble { reranked: true, })); - return client; + return bench; } function captureGuidedQueryHandler(): (params: { @@ -213,10 +217,10 @@ function createBenchPineconeMock(): PineconeClient { content: String(h.fields['chunk_text'] ?? ''), score: h._score, metadata: { - document_number: h.fields['document_number'], - title: h.fields['title'], - url: h.fields['url'], - author: h.fields['author'], + document_number: String(h.fields['document_number'] ?? ''), + title: String(h.fields['title'] ?? ''), + url: String(h.fields['url'] ?? ''), + author: String(h.fields['author'] ?? ''), }, reranked: false, })); @@ -251,6 +255,10 @@ function createBenchPineconeMock(): PineconeClient { } async function main(): Promise { + if (SMOKE) { + process.env['PINECONE_API_KEY'] ??= 'bench-key'; + process.env['PINECONE_INDEX_NAME'] ??= 'bench-index'; + } setLogLevel('ERROR'); const results: BenchmarkResult[] = []; @@ -266,16 +274,19 @@ async function main(): Promise { }) ); - results.push( - await runBenchmark('query_with_rerank', async () => { - await queryClient.query({ - query: 'benchmark hybrid query text', - namespace: 'docs', - topK: TOP_K, - useReranking: true, - }); - }) - ); + // Skipped in smoke: rerank double is dead post-restructure; would hit real Pinecone HTTP. + if (!SMOKE) { + results.push( + await runBenchmark('query_with_rerank', async () => { + await queryClient.query({ + query: 'benchmark hybrid query text', + namespace: 'docs', + topK: TOP_K, + useReranking: true, + }); + }) + ); + } setPineconeClient(createBenchPineconeMock()); invalidateNamespacesCache(); @@ -320,11 +331,13 @@ async function main(): Promise { results, }; - const __filename = fileURLToPath(import.meta.url); - const __dirname = dirname(__filename); - const baselinePath = join(__dirname, 'baseline.json'); - writeFileSync(baselinePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); - console.log(`Wrote ${baselinePath}`); + if (!SMOKE) { + const __filename = fileURLToPath(import.meta.url); + const __dirname = dirname(__filename); + const baselinePath = join(__dirname, 'baseline.json'); + writeFileSync(baselinePath, `${JSON.stringify(payload, null, 2)}\n`, 'utf8'); + console.log(`Wrote ${baselinePath}`); + } } main().catch(exitOnDemoFailure('latency benchmark')); diff --git a/benchmarks/tsconfig.json b/benchmarks/tsconfig.json new file mode 100644 index 0000000..bb5d680 --- /dev/null +++ b/benchmarks/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "noEmit": true, + "rootDir": ".." + }, + "include": ["./**/*.ts", "../src/**/*.ts"], + "exclude": ["../node_modules", "../dist", "../**/*.test.ts", "../src/core/server/tools/test-helpers.ts"] +} diff --git a/package.json b/package.json index 09dfb75..96d6832 100644 --- a/package.json +++ b/package.json @@ -59,6 +59,7 @@ "test:coverage": "vitest run --coverage", "test:watch": "vitest", "benchmark": "tsx benchmarks/latency.ts", + "benchmark:smoke": "tsx benchmarks/latency.ts --smoke", "test:search": "tsx scripts/test-search.ts", "lint": "eslint src/", "lint:fix": "eslint src/ --fix", @@ -66,7 +67,8 @@ "format:check": "prettier --check \"src/**/*.ts\" \"*.json\" \".prettierrc\"", "docs:link-check": "node scripts/docs-link-check.mjs", "typecheck": "tsc --noEmit", - "ci": "npm run typecheck && npm run lint && npm run format:check && npm run build && npm run test:coverage", + "typecheck:benchmarks": "tsc -p benchmarks/tsconfig.json", + "ci": "npm run typecheck && npm run typecheck:benchmarks && npm run lint && npm run format:check && npm run build && npm run benchmark:smoke && npm run test:coverage", "release:check": "npm run ci && npm pack --dry-run --ignore-scripts", "ci:local": "bash scripts/ci-local.sh", "prepublishOnly": "npm run ci",