feat: add harness optimization primitives (#6)

* feat(0.7.1): SKILL.md as sole doc + SubprocessSandboxDriver constructor fallbacks

Motivation: meta-analysis of starter-foundry's Gen 6→Round-0-post-Gen-9
arc surfaced 10+ incident-driven lessons about using this package. They
lived nowhere canonical (README/CLAUDE.md described a stale 0.2-era API
surface; the actual v0.7 builder-of-builders + sandbox harness exports
had zero usage docs). Two shipped bugs traced to the same driver
construct-vs-call cwd footgun. Consolidating into one authoritative
doc + closing the footgun at the source.

Changes:

- .claude/skills/agent-eval/SKILL.md (NEW, sole source of truth)
  - minimal builder-of-builders path
  - 4 footguns (cwd-in-constructor, fallback-to-pass, fidelity-without-
    compile-gate, blob-vs-files channel)
  - 3 rules (both gates, single-source dispatch, Phase 1.5 walks entry
    points)
  - three-layer eval contract (builder → app-build → app-runtime)
  - regression tests every consumer should carry
  - extend-don't-duplicate index over the 100+ exports
  - muffled-gate pattern catalog (7 sub-shapes from shipped bugs)

- README.md + CLAUDE.md → pointers to SKILL.md. No duplicated content.

- SubprocessSandboxDriver constructor now accepts `{cwd?, env?}` as
  FALLBACKS when HarnessConfig omits them. Per-call config always wins.
  Pre-0.7.1 the constructor took no declared args, so TS tolerated
  `new Driver({cwd})` and silently dropped the arg at runtime — the
  exact shape of the Gen 8b promoter + Round-0 runtime eval bugs in
  starter-foundry. 0.7.1 makes the natural misuse do the obvious thing.
  New type: `SubprocessDriverDefaults`. Zero breaking changes for
  code that already reads cwd from HarnessConfig (the documented path).

- tests/sandbox-harness.test.ts: +3 tests guarding the new defaults
  contract — default.cwd honored, per-call wins over default,
  defaults.env merges correctly.

322/322 tests pass (was 319; +3 new). typecheck clean.
Version: 0.7.0 → 0.7.1.

* feat: add harness optimization primitives

Author: drewstone

src/harness-optimizer.ts

@@ -0,0 +1,216 @@
+import { paretoFrontier, type Objective, type ParetoResult } from './pareto'
+import { aggregateRunScore, type RunScore, type RunScoreWeights } from './run-score'
+import { RunCritic, type RunTrace } from './run-critic'
+import type { SteeringBundle } from './steering'
+
+export type HarnessIntervention =
+  | 'continue'
+  | 'plan'
+  | 'audit'
+  | 'recover'
+  | 'repair'
+  | 'verify'
+  | 'final_gate'
+  | 'wait_for_measurement'
+  | 'abort'
+
+export interface WorkflowTopology {
+  id: string
+  interventions: HarnessIntervention[]
+  maxParallelBranches?: number
+  metadata?: Record<string, unknown>
+}
+
+export interface MeasurementPolicy {
+  required: string[]
+  optional?: string[]
+  promoteOn?: Array<keyof RunScore | 'aggregate'>
+}
+
+export interface HarnessVariant {
+  id: string
+  steering?: SteeringBundle
+  topology?: WorkflowTopology
+  measurement?: MeasurementPolicy
+  budgets?: Record<string, number>
+  models?: Record<string, string>
+  reviewers?: Record<string, string>
+  metadata?: Record<string, unknown>
+}
+
+export interface HarnessScenario {
+  id: string
+  task: string
+  split?: 'train' | 'validation' | 'test' | string
+  metadata?: Record<string, unknown>
+}
+
+export interface HarnessRunRequest {
+  variant: HarnessVariant
+  scenario: HarnessScenario
+  trialIndex: number
+}
+
+export interface HarnessAdapter {
+  run(request: HarnessRunRequest): Promise<RunTrace>
+}
+
+export interface HarnessRunResult {
+  variant: HarnessVariant
+  scenario: HarnessScenario
+  trialIndex: number
+  trace: RunTrace
+  score: RunScore
+  aggregate: number
+}
+
+export interface HarnessVariantReport {
+  variant: HarnessVariant
+  runs: HarnessRunResult[]
+  aggregateMean: number
+  passRate: number
+  costUsdMean: number
+  wallSecondsMean: number
+  scoreMean: RunScore
+}
+
+export interface HarnessSelection {
+  winner: HarnessVariantReport
+  frontier: ParetoResult<HarnessVariantReport>
+  reports: HarnessVariantReport[]
+}
+
+export interface HarnessExperimentResult {
+  results: HarnessRunResult[]
+  selection: HarnessSelection
+}
+
+export interface HarnessExperimentConfig {
+  adapter: HarnessAdapter
+  variants: HarnessVariant[]
+  scenarios: HarnessScenario[]
+  trialsPerScenario?: number
+  parallelism?: number
+  weights?: Partial<RunScoreWeights>
+  objectives?: Array<Objective<HarnessVariantReport>>
+  score?: (trace: RunTrace, request: HarnessRunRequest) => RunScore | Promise<RunScore>
+  onResult?: (result: HarnessRunResult) => void | Promise<void>
+}
+
+export const DEFAULT_HARNESS_OBJECTIVES: Array<Objective<HarnessVariantReport>> = [
+  { name: 'aggregate', direction: 'maximize', value: (r) => r.aggregateMean },
+  { name: 'pass_rate', direction: 'maximize', value: (r) => r.passRate },
+  { name: 'cost', direction: 'minimize', value: (r) => r.costUsdMean },
+  { name: 'wall', direction: 'minimize', value: (r) => r.wallSecondsMean },
+]
+
+export async function runHarnessExperiment(config: HarnessExperimentConfig): Promise<HarnessExperimentResult> {
+  const jobs = buildJobs(config)
+  const critic = new RunCritic({ weights: config.weights })
+  const score = config.score ?? ((trace: RunTrace) => critic.scoreTrace(trace))
+  const results = await mapLimit(jobs, config.parallelism ?? 1, async (request) => {
+    const trace = await config.adapter.run(request)
+    const runScore = await score(trace, request)
+    const result: HarnessRunResult = {
+      variant: request.variant,
+      scenario: request.scenario,
+      trialIndex: request.trialIndex,
+      trace,
+      score: runScore,
+      aggregate: aggregateRunScore(runScore, config.weights),
+    }
+    await config.onResult?.(result)
+    return result
+  })
+  return { results, selection: selectHarnessVariant(results, config.objectives) }
+}
+
+export function selectHarnessVariant(
+  results: HarnessRunResult[],
+  objectives: Array<Objective<HarnessVariantReport>> = DEFAULT_HARNESS_OBJECTIVES,
+): HarnessSelection {
+  const reports = summarizeHarnessResults(results)
+  if (reports.length === 0) throw new Error('selectHarnessVariant: no results')
+  const frontier = paretoFrontier(reports, objectives)
+  const candidates = frontier.frontier.length ? frontier.frontier : reports
+  const winner = [...candidates].sort((a, b) => b.aggregateMean - a.aggregateMean)[0]
+  if (!winner) throw new Error('selectHarnessVariant: no winner')
+  return { winner, frontier, reports }
+}
+
+export function summarizeHarnessResults(results: HarnessRunResult[]): HarnessVariantReport[] {
+  const byVariant = new Map<string, HarnessRunResult[]>()
+  for (const result of results) {
+    byVariant.set(result.variant.id, [...(byVariant.get(result.variant.id) ?? []), result])
+  }
+  return [...byVariant.values()]
+    .map((runs) => {
+      const variant = runs[0]?.variant
+      if (!variant) throw new Error('summarizeHarnessResults: empty variant bucket')
+      return {
+        variant,
+        runs,
+        aggregateMean: mean(runs.map((r) => r.aggregate)),
+        passRate: mean(runs.map((r) => r.score.success)),
+        costUsdMean: mean(runs.map((r) => r.score.costUsd)),
+        wallSecondsMean: mean(runs.map((r) => r.score.wallSeconds)),
+        scoreMean: meanRunScore(runs.map((r) => r.score)),
+      }
+    })
+    .sort((a, b) => b.aggregateMean - a.aggregateMean)
+}
+
+function buildJobs(config: HarnessExperimentConfig): HarnessRunRequest[] {
+  if (config.variants.length === 0) throw new Error('runHarnessExperiment: at least one variant required')
+  if (config.scenarios.length === 0) throw new Error('runHarnessExperiment: at least one scenario required')
+  const trials = Math.max(1, Math.floor(config.trialsPerScenario ?? 1))
+  const jobs: HarnessRunRequest[] = []
+  for (const variant of config.variants) {
+    for (const scenario of config.scenarios) {
+      for (let trialIndex = 0; trialIndex < trials; trialIndex++) {
+        jobs.push({ variant, scenario, trialIndex })
+      }
+    }
+  }
+  return jobs
+}
+
+async function mapLimit<T, R>(
+  items: T[],
+  limit: number,
+  fn: (item: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length)
+  let next = 0
+  const workerCount = Math.max(1, Math.min(Math.floor(limit), items.length))
+  await Promise.all(Array.from({ length: workerCount }, async () => {
+    while (next < items.length) {
+      const index = next++
+      const item = items[index]
+      if (item === undefined) continue
+      results[index] = await fn(item)
+    }
+  }))
+  return results
+}
+
+function mean(values: number[]): number {
+  return values.length ? values.reduce((sum, value) => sum + value, 0) / values.length : 0
+}
+
+function meanRunScore(scores: RunScore[]): RunScore {
+  return {
+    success: mean(scores.map((s) => s.success)),
+    goalProgress: mean(scores.map((s) => s.goalProgress)),
+    repoGroundedness: mean(scores.map((s) => s.repoGroundedness)),
+    driftPenalty: mean(scores.map((s) => s.driftPenalty)),
+    toolUseQuality: mean(scores.map((s) => s.toolUseQuality)),
+    patchQuality: mean(scores.map((s) => s.patchQuality)),
+    testReality: mean(scores.map((s) => s.testReality)),
+    finalGate: mean(scores.map((s) => s.finalGate)),
+    reviewerBlockers: mean(scores.map((s) => s.reviewerBlockers)),
+    costUsd: mean(scores.map((s) => s.costUsd)),
+    wallSeconds: mean(scores.map((s) => s.wallSeconds)),
+    notes: scores.flatMap((s) => s.notes ?? []),
+  }
+}

src/index.ts

@@ -138,6 +138,26 @@ export type {
   SteeringOptimizerConfig,
   AxSteeringOptimizerConfig,
 } from './steering-optimizer'
+export {
+  DEFAULT_HARNESS_OBJECTIVES,
+  runHarnessExperiment,
+  selectHarnessVariant,
+  summarizeHarnessResults,
+} from './harness-optimizer'
+export type {
+  HarnessAdapter,
+  HarnessExperimentConfig,
+  HarnessExperimentResult,
+  HarnessIntervention,
+  HarnessRunRequest,
+  HarnessRunResult,
+  HarnessScenario,
+  HarnessSelection,
+  HarnessVariant,
+  HarnessVariantReport,
+  MeasurementPolicy,
+  WorkflowTopology,
+} from './harness-optimizer'
 export {
   JudgeRunner,
   runJudgeFleet,

src/run-critic.ts

@@ -50,6 +50,9 @@ export class RunCritic {
     const toolSpans = trace.spans.filter((s): s is Extract<Span, { kind: 'tool' }> => s.kind === 'tool')
     const judgeSpans = trace.spans.filter((s): s is Extract<Span, { kind: 'judge' }> => s.kind === 'judge')
     const sandboxSpans = trace.spans.filter((s): s is Extract<Span, { kind: 'sandbox' }> => s.kind === 'sandbox')
+    const finalGateSpans = judgeSpans.filter((span) =>
+      span.dimension === 'final_gate' || span.attributes?.finalGate === true,
+    )
 
     const success = trace.run.outcome?.pass === true ? 1 : trace.run.status === 'completed' ? 0.5 : 0
     if (!success) notes.push('run did not complete with pass=true')
@@ -78,6 +81,17 @@ export class RunCritic {
         : 0
     if (!testReality) notes.push('no real test/build evidence recorded')
 
+    const blockerSpans = judgeSpans.filter((span) =>
+      isBlockingJudge(span),
+    )
+    const finalGateBlockers = finalGateSpans.filter((span) => isBlockingJudge(span))
+    const finalGate = finalGateSpans.length ? (finalGateBlockers.length ? 0 : 1) : success
+    if (finalGateBlockers.length) notes.push(`final gate blocked by ${finalGateBlockers.length} reviewer(s)`)
+    else if (!finalGateSpans.length) notes.push('no final gate judgment recorded')
+
+    const reviewerBlockers = judgeSpans.length ? blockerSpans.length / judgeSpans.length : 0
+    if (reviewerBlockers) notes.push(`detected ${blockerSpans.length} blocking reviewer signal(s)`)
+
     const positiveGroundingSignals =
       patchEvidence +
       sandboxSpans.length +
@@ -108,6 +122,8 @@ export class RunCritic {
       toolUseQuality,
       patchQuality,
       testReality,
+      finalGate,
+      reviewerBlockers,
       costUsd,
       wallSeconds,
       notes,
@@ -130,3 +146,15 @@ function normalizeJudgeScore(score: number): number {
 function looksRepoGrounded(text: string): boolean {
   return /(?:src\/|tests?\/|package\.json|tsconfig|\.ts\b|\.tsx\b|git status|pnpm |npm |vitest|pytest|jest)/i.test(text)
 }
+
+function isBlockingJudge(span: Extract<Span, { kind: 'judge' }>): boolean {
+  return span.attributes?.blocking === true ||
+    span.attributes?.verdict === 'BLOCKING' ||
+    positiveNumber(span.attributes?.blockingFindings) ||
+    positiveNumber(span.attributes?.highFindings) ||
+    span.score <= 2
+}
+
+function positiveNumber(value: unknown): boolean {
+  return typeof value === 'number' && value > 0
+}

src/run-score.ts

@@ -6,6 +6,8 @@ export interface RunScore {
   toolUseQuality: number
   patchQuality: number
   testReality: number
+  finalGate: number
+  reviewerBlockers: number
   costUsd: number
   wallSeconds: number
   notes?: string[]
@@ -19,6 +21,8 @@ export interface RunScoreWeights {
   toolUseQuality: number
   patchQuality: number
   testReality: number
+  finalGate: number
+  reviewerBlockers: number
   costUsd: number
   wallSeconds: number
 }
@@ -31,6 +35,8 @@ export const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights = {
   toolUseQuality: 1,
   patchQuality: 1.25,
   testReality: 1.5,
+  finalGate: 3,
+  reviewerBlockers: -2,
   costUsd: -0.2,
   wallSeconds: -0.1,
 }
@@ -48,6 +54,8 @@ export function aggregateRunScore(
     w.toolUseQuality * clamp01(score.toolUseQuality) +
     w.patchQuality * clamp01(score.patchQuality) +
     w.testReality * clamp01(score.testReality) +
+    w.finalGate * clamp01(score.finalGate) +
+    w.reviewerBlockers * clamp01(score.reviewerBlockers) +
     w.costUsd * Math.max(0, score.costUsd) +
     w.wallSeconds * Math.max(0, score.wallSeconds / 60)
   )