feat(03-03): centralize eval harness scoring logic

- add reusable src/eval harness and shared types for fixture scoring
- migrate eval harness tests to import shared module and validate both fixtures

Author: PatrickSys

src/eval/harness.ts

@@ -0,0 +1,218 @@
+import crypto from 'crypto';
+import type { SearchResult } from '../types/index.js';
+import type {
+  EvalGate,
+  EvalQuery,
+  EvalResult,
+  EvalSummary,
+  EvaluateFixtureParams,
+  FormatEvalReportParams
+} from './types.js';
+
+function normalizePath(filePath: string): string {
+  return filePath.toLowerCase().replace(/\\/g, '/');
+}
+
+function isTestFile(filePath: string): boolean {
+  const normalized = normalizePath(filePath);
+  return (
+    normalized.includes('.spec.') ||
+    normalized.includes('.test.') ||
+    normalized.includes('/e2e/') ||
+    normalized.includes('/__tests__/')
+  );
+}
+
+function matchesPattern(filePath: string, patterns: string[]): boolean {
+  const normalized = normalizePath(filePath);
+  return patterns.some((pattern) => normalized.includes(pattern.toLowerCase()));
+}
+
+function dedupeByFile(results: SearchResult[]): SearchResult[] {
+  const seen = new Set<string>();
+  const deduped: SearchResult[] = [];
+
+  for (const result of results) {
+    const key = normalizePath(result.filePath);
+    if (seen.has(key)) {
+      continue;
+    }
+
+    seen.add(key);
+    deduped.push(result);
+  }
+
+  return deduped;
+}
+
+function getExpectedPatterns(query: EvalQuery): string[] {
+  return query.expectedPatterns ?? query.expectedTopFiles ?? [];
+}
+
+function getExpectedNotPatterns(query: EvalQuery): string[] {
+  return query.expectedNotPatterns ?? query.expectedNotTopFiles ?? [];
+}
+
+function evaluateQuery(query: EvalQuery, results: SearchResult[]): EvalResult {
+  const uniqueResults = dedupeByFile(results);
+  const topFile = uniqueResults.length > 0 ? uniqueResults[0].filePath : null;
+  const top3Files = uniqueResults.slice(0, 3).map((result) => result.filePath);
+
+  const expectedPatterns = getExpectedPatterns(query);
+  const expectedNotPatterns = getExpectedNotPatterns(query);
+
+  const top1Correct =
+    topFile !== null &&
+    matchesPattern(topFile, expectedPatterns) &&
+    !matchesPattern(topFile, expectedNotPatterns);
+
+  const top3Recall = top3Files.some(
+    (filePath) =>
+      matchesPattern(filePath, expectedPatterns) && !matchesPattern(filePath, expectedNotPatterns)
+  );
+
+  const specCount = top3Files.filter((filePath) => isTestFile(filePath)).length;
+  const specContaminated = specCount >= 2;
+
+  return {
+    queryId: query.id,
+    query: query.query,
+    category: query.category,
+    expectedPatterns,
+    expectedNotPatterns,
+    topFile,
+    top3Files,
+    top1Correct,
+    top3Recall,
+    specContaminated,
+    score: uniqueResults.length > 0 ? uniqueResults[0].score : 0
+  };
+}
+
+function resolveGateThreshold(total: number, gate: EvalGate): number {
+  if (gate <= 1) {
+    return Math.ceil(total * gate);
+  }
+
+  return Math.ceil(gate);
+}
+
+function hashPath(filePath: string): string {
+  return crypto.createHash('sha1').update(normalizePath(filePath)).digest('hex').slice(0, 8);
+}
+
+function formatPath(filePath: string | null, redactPaths: boolean): string {
+  if (!filePath) {
+    return 'none';
+  }
+
+  const normalized = filePath.replace(/\\/g, '/');
+  if (!redactPaths) {
+    return normalized;
+  }
+
+  const base = normalized.split('/').pop() || normalized;
+  return `path#${hashPath(normalized)}/${base}`;
+}
+
+export async function evaluateFixture({
+  fixture,
+  searcher,
+  limit = 5,
+  searchOptions
+}: EvaluateFixtureParams): Promise<EvalSummary> {
+  const results: EvalResult[] = [];
+
+  for (const query of fixture.queries) {
+    const searchResults = await searcher.search(query.query, limit, undefined, searchOptions);
+    results.push(evaluateQuery(query, searchResults));
+  }
+
+  return summarizeEvaluation(results);
+}
+
+export function summarizeEvaluation(results: EvalResult[], gate: EvalGate = 0.7): EvalSummary {
+  const total = results.length;
+  const top1Correct = results.filter((result) => result.top1Correct).length;
+  const top3RecallCount = results.filter((result) => result.top3Recall).length;
+  const specContaminatedCount = results.filter((result) => result.specContaminated).length;
+  const avgTopScore =
+    total > 0 ? results.reduce((sum, result) => sum + result.score, 0) / total : 0;
+  const gateThreshold = resolveGateThreshold(total, gate);
+
+  return {
+    total,
+    top1Correct,
+    top1Accuracy: total > 0 ? top1Correct / total : 0,
+    top3RecallCount,
+    top3Recall: total > 0 ? top3RecallCount / total : 0,
+    specContaminatedCount,
+    specContaminationRate: total > 0 ? specContaminatedCount / total : 0,
+    avgTopScore,
+    gateThreshold,
+    passesGate: total > 0 && top1Correct >= gateThreshold,
+    results
+  };
+}
+
+export function formatEvalReport({
+  codebaseLabel,
+  fixturePath,
+  summary,
+  redactPaths = true
+}: FormatEvalReportParams): string {
+  const lines: string[] = [];
+  const wins = summary.results.filter((result) => result.top1Correct);
+  const failures = summary.results.filter((result) => !result.top1Correct);
+
+  lines.push(`\n=== Eval Report: ${codebaseLabel} ===`);
+  lines.push(`Fixture: ${fixturePath}`);
+  lines.push(
+    `Top-1 Accuracy: ${summary.top1Correct}/${summary.total} (${(summary.top1Accuracy * 100).toFixed(0)}%)`
+  );
+  lines.push(
+    `Top-3 Recall:   ${summary.top3RecallCount}/${summary.total} (${(summary.top3Recall * 100).toFixed(0)}%)`
+  );
+  lines.push(
+    `Spec Contamination: ${summary.specContaminatedCount}/${summary.total} (${(summary.specContaminationRate * 100).toFixed(0)}%)`
+  );
+  lines.push(
+    `Gate (${summary.gateThreshold}/${summary.total}): ${summary.passesGate ? 'PASS' : 'FAIL'}`
+  );
+  lines.push(`Wins: ${wins.length} | Failures: ${failures.length}`);
+
+  lines.push('\nWins:');
+  if (wins.length === 0) {
+    lines.push('  (none)');
+  } else {
+    for (const result of wins) {
+      lines.push(
+        `  PASS #${result.queryId} [${result.category}] "${result.query}" -> ${formatPath(result.topFile, redactPaths)} (${result.score.toFixed(3)})`
+      );
+    }
+  }
+
+  lines.push('\nFailures:');
+  if (failures.length === 0) {
+    lines.push('  (none)');
+  } else {
+    for (const result of failures) {
+      lines.push(
+        `  FAIL #${result.queryId} [${result.category}] "${result.query}" -> ${formatPath(result.topFile, redactPaths)} (${result.score.toFixed(3)})`
+      );
+      lines.push(`    expected: ${result.expectedPatterns.join(' | ') || '(none)'}`);
+      lines.push(`    expected-not: ${result.expectedNotPatterns.join(' | ') || '(none)'}`);
+      lines.push('    top-3 actual:');
+      if (result.top3Files.length === 0) {
+        lines.push('      1. none');
+      } else {
+        for (let index = 0; index < result.top3Files.length; index++) {
+          lines.push(`      ${index + 1}. ${formatPath(result.top3Files[index], redactPaths)}`);
+        }
+      }
+    }
+  }
+
+  lines.push('\n================================');
+  return lines.join('\n');
+}

src/eval/types.ts

@@ -0,0 +1,65 @@
+import type { SearchOptions, CodebaseSearcher } from '../core/search.js';
+
+export interface EvalQuery {
+  id: number;
+  query: string;
+  category: string;
+  expectedPatterns?: string[];
+  expectedNotPatterns?: string[];
+  expectedTopFiles?: string[];
+  expectedNotTopFiles?: string[];
+  notes?: string;
+}
+
+export interface EvalFixture {
+  description?: string;
+  codebase?: string;
+  repository?: string;
+  frozenDate?: string;
+  notes?: string;
+  queries: EvalQuery[];
+}
+
+export interface EvalResult {
+  queryId: number;
+  query: string;
+  category: string;
+  expectedPatterns: string[];
+  expectedNotPatterns: string[];
+  topFile: string | null;
+  top3Files: string[];
+  top1Correct: boolean;
+  top3Recall: boolean;
+  specContaminated: boolean;
+  score: number;
+}
+
+export interface EvalSummary {
+  total: number;
+  top1Correct: number;
+  top1Accuracy: number;
+  top3RecallCount: number;
+  top3Recall: number;
+  specContaminatedCount: number;
+  specContaminationRate: number;
+  avgTopScore: number;
+  gateThreshold: number;
+  passesGate: boolean;
+  results: EvalResult[];
+}
+
+export interface EvaluateFixtureParams {
+  fixture: EvalFixture;
+  searcher: CodebaseSearcher;
+  limit?: number;
+  searchOptions?: SearchOptions;
+}
+
+export type EvalGate = number;
+
+export interface FormatEvalReportParams {
+  codebaseLabel: string;
+  fixturePath: string;
+  summary: EvalSummary;
+  redactPaths?: boolean;
+}