fix: recalibrate success criteria thresholds for correction-aware metrics

First-attempt now means zero corrections, which is stricter than before.
Lower threshold to 30% (aspirational), add withCorrectionPassRate at 90%
as the primary quality gate, keep withRetryPassRate at 95%.

Author: nicknisi

tests/evals/success-criteria.spec.ts

@@ -2,65 +2,74 @@ import { describe, it, expect } from 'vitest';
 import { validateResults, DEFAULT_CRITERIA, type SuccessCriteria } from './success-criteria.js';
 import type { EvalResult } from './types.js';
 
-function makeResult(passed: boolean, attempts: number = 1): EvalResult {
+function makeResult(passed: boolean, attempts: number = 1, correctionAttempts: number = 0): EvalResult {
   return {
     scenario: `test-${Math.random().toString(36).slice(2)}`,
     passed,
     duration: 1000,
     attempts,
+    correctionAttempts,
   };
 }
 
 describe('success-criteria', () => {
   describe('DEFAULT_CRITERIA', () => {
     it('has expected default thresholds', () => {
-      expect(DEFAULT_CRITERIA.firstAttemptPassRate).toBe(0.9);
+      expect(DEFAULT_CRITERIA.firstAttemptPassRate).toBe(0.3);
+      expect(DEFAULT_CRITERIA.withCorrectionPassRate).toBe(0.9);
       expect(DEFAULT_CRITERIA.withRetryPassRate).toBe(0.95);
     });
   });
 
   describe('validateResults', () => {
     it('returns passed=true when all criteria met', () => {
-      // 10 results, 9 passed on first attempt, 1 passed on retry
+      // 10 results: 4 clean (40% > 30%), 5 corrected (9/10 = 90% correction), 1 retried (100% retry)
       const results: EvalResult[] = [
-        ...Array(9)
+        ...Array(4)
           .fill(null)
-          .map(() => makeResult(true, 1)),
+          .map(() => makeResult(true, 1, 0)),
+        ...Array(5)
+          .fill(null)
+          .map(() => makeResult(true, 1, 1)),
         makeResult(true, 2),
       ];
 
       const validation = validateResults(results);
 
       expect(validation.passed).toBe(true);
       expect(validation.failures).toHaveLength(0);
-      expect(validation.actual.firstAttemptPassRate).toBe(0.9);
+      expect(validation.actual.firstAttemptPassRate).toBe(0.4);
+      expect(validation.actual.withCorrectionPassRate).toBe(0.9);
       expect(validation.actual.withRetryPassRate).toBe(1);
     });
 
     it('returns passed=false when first-attempt rate below threshold', () => {
-      // 10 results, only 8 passed on first attempt
+      // 10 results, only 2 passed on first attempt (20% < 30% threshold)
       const results: EvalResult[] = [
-        ...Array(8)
+        ...Array(2)
           .fill(null)
           .map(() => makeResult(true, 1)),
-        makeResult(true, 2),
+        ...Array(7)
+          .fill(null)
+          .map(() => makeResult(true, 2)),
         makeResult(true, 2),
       ];
 
       const validation = validateResults(results);
 
       expect(validation.passed).toBe(false);
-      expect(validation.failures).toHaveLength(1);
-      expect(validation.failures[0]).toContain('First-attempt');
-      expect(validation.failures[0]).toContain('80.0%');
+      expect(validation.failures.some((f) => f.includes('First-attempt'))).toBe(true);
     });
 
     it('returns passed=false when with-retry rate below threshold', () => {
-      // 10 results, 9 passed first attempt, 1 failed entirely
+      // 10 results: 4 clean, 5 corrected (90% correction), 1 failed → 90% retry < 95%
       const results: EvalResult[] = [
-        ...Array(9)
+        ...Array(4)
           .fill(null)
-          .map(() => makeResult(true, 1)),
+          .map(() => makeResult(true, 1, 0)),
+        ...Array(5)
+          .fill(null)
+          .map(() => makeResult(true, 1, 1)),
         makeResult(false, 3),
       ];
 
@@ -71,21 +80,24 @@ describe('success-criteria', () => {
       expect(validation.failures[0]).toContain('With-retry');
     });
 
-    it('returns both failures when both criteria not met', () => {
-      // 10 results, 7 passed first attempt, 1 failed
+    it('returns both failures when multiple criteria not met', () => {
+      // 10 results, 2 passed first attempt (20% < 30%), 4 failed entirely (60% < 95% retry)
       const results: EvalResult[] = [
-        ...Array(7)
+        ...Array(2)
           .fill(null)
           .map(() => makeResult(true, 1)),
-        makeResult(true, 2),
-        makeResult(true, 2),
-        makeResult(false, 3),
+        ...Array(4)
+          .fill(null)
+          .map(() => makeResult(true, 2)),
+        ...Array(4)
+          .fill(null)
+          .map(() => makeResult(false, 3)),
       ];
 
       const validation = validateResults(results);
 
       expect(validation.passed).toBe(false);
-      expect(validation.failures).toHaveLength(2);
+      expect(validation.failures.length).toBeGreaterThanOrEqual(2);
     });
 
     it('handles empty results array', () => {
@@ -120,11 +132,18 @@ describe('success-criteria', () => {
     });
 
     it('passes when exactly at threshold', () => {
-      // Exactly 90% first-attempt, 95% with-retry
+      // 20 results:
+      //   6 clean first-attempt (attempt=1, corrections=0) → 30% first-attempt
+      //  12 self-corrected (attempt=1, corrections=1)       → 18/20 = 90% with-correction
+      //   1 passed on scenario retry (attempt=2)             → 19/20 = 95% with-retry
+      //   1 failed (attempt=3)
       const results: EvalResult[] = [
-        ...Array(18)
+        ...Array(6)
           .fill(null)
-          .map(() => makeResult(true, 1)),
+          .map(() => makeResult(true, 1, 0)),
+        ...Array(12)
+          .fill(null)
+          .map(() => makeResult(true, 1, 1)),
         makeResult(true, 2),
         makeResult(false, 3),
       ];

tests/evals/success-criteria.ts

@@ -17,7 +17,8 @@ export interface SuccessCriteria {
 
 /** Default thresholds for CI enforcement */
 export const DEFAULT_CRITERIA: SuccessCriteria = {
-  firstAttemptPassRate: 0.9,
+  firstAttemptPassRate: 0.3,
+  withCorrectionPassRate: 0.9,
   withRetryPassRate: 0.95,
 };