fix: align discovery protocol metrics

Author: PatrickSys

scripts/run-eval.mjs

@@ -285,7 +285,6 @@ async function main() {
   });
 
   const summaries = [summaryA];
-  let passesAllGates = summaryA.passesGate;
 
   if (codebaseB) {
     const summaryB = await runSingleEvaluation({
@@ -296,8 +295,6 @@ async function main() {
     });
 
     summaries.push(summaryB);
-    passesAllGates =
-      mode === 'discovery' ? passesAllGates : passesAllGates && summaryB.passesGate;
   }
 
   if (mode === 'discovery') {
@@ -320,6 +317,7 @@ async function main() {
     process.exit(gate.status === 'failed' ? 1 : 0);
   }
 
+  const passesAllGates = summaries.every((summary) => summary.passesGate);
   printCombinedSummary(summaries, mode);
   process.exit(passesAllGates ? 0 : 1);
 }

src/eval/discovery-harness.ts

@@ -240,7 +240,10 @@ function compareMetric(
   comparatorValue: number | null,
   metric: DiscoveryMetricName
 ): DiscoveryMetricComparison {
-  const lowerIsBetter = metric === 'averageEstimatedTokens' || metric === 'averageFirstRelevantHit';
+  const lowerIsBetter =
+    metric === 'averagePayloadBytes' ||
+    metric === 'averageEstimatedTokens' ||
+    metric === 'averageFirstRelevantHit';
   const passes =
     actualValue !== null &&
     comparatorValue !== null &&
@@ -260,7 +263,10 @@ function compareMetricWithinTolerance(
   metric: DiscoveryMetricName,
   tolerancePercent: number
 ): DiscoveryMetricComparison {
-  const lowerIsBetter = metric === 'averageFirstRelevantHit';
+  const lowerIsBetter =
+    metric === 'averagePayloadBytes' ||
+    metric === 'averageEstimatedTokens' ||
+    metric === 'averageFirstRelevantHit';
   const multiplier = 1 + tolerancePercent / 100;
   const passes =
     actualValue !== null &&

src/eval/types.ts

@@ -151,6 +151,7 @@ export interface DiscoverySurfaceResult {
 
 export type DiscoveryMetricName =
   | 'averageUsefulness'
+  | 'averagePayloadBytes'
   | 'averageEstimatedTokens'
   | 'averageFirstRelevantHit'
   | 'bestExampleUsefulnessRate';
@@ -198,6 +199,7 @@ export interface DiscoveryBenchmarkProtocol {
 
 export interface DiscoveryComparatorMetrics {
   averageUsefulness?: number | null;
+  averagePayloadBytes?: number | null;
   averageEstimatedTokens?: number | null;
   averageFirstRelevantHit?: number | null;
   bestExampleUsefulnessRate?: number | null;

tests/fixtures/discovery-benchmark-protocol.json

@@ -39,8 +39,8 @@
     }
   ],
   "metrics": {
-    "payloadCost": ["payloadBytes", "estimatedTokens"],
-    "usefulness": ["usefulnessScore", "firstRelevantHit", "bestExampleUseful"]
+    "payloadCost": ["averagePayloadBytes", "averageEstimatedTokens"],
+    "usefulness": ["averageUsefulness", "averageFirstRelevantHit", "bestExampleUsefulnessRate"]
   },
   "fairnessRules": [
     "Use only current shipped codebase-context surfaces in the direct-tool lane.",