feat(evals): wire WebTailBench through verifier (browserbase#2135)

# Why

The eval harness needs one reusable path for running an agent, recording
its trajectory, and scoring it with the verifier. This PR applies that
path to WebTailBench first as the narrowest dataset migration.

# What Changed

- Added `runWithVerifier` for rubric resolution, trajectory recording,
verifier scoring, and result persistence.
- Migrated the WebTailBench task path to verifier-backed scoring.
- Added validated success-mode conversion from verifier verdicts;
invalid or missing values fall back to `outcome`.
- Added focused tests for success-mode validation.
- Added a focused WebTailBench verifier smoke script.
- Kept verifier usage explicit with `backend: "verifier"`.
- Removed upstream verifier references from comments.

# Tests

- `pnpm --filter @browserbasehq/stagehand run typecheck`
- `pnpm --filter @browserbasehq/stagehand-evals run typecheck`
- `pnpm --dir packages/evals exec vitest run
tests/framework/rubricCache.test.ts
tests/framework/verifierAdapter.test.ts tests/tui/commandTree.test.ts
tests/framework/claudeCodeRunner.test.ts
tests/framework/codexRunner.test.ts tests/cli.test.ts`
- `git diff --check`

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: miguelg719

packages/evals/datasets/webtailbench/WebTailBench_data.jsonl

@@ -0,0 +1,27 @@
+/**
+ * adHocRubric — synthesize a Rubric from one or more natural-language
+ * criteria without invoking the LLM-based rubric generator.
+ *
+ * Used by migrated custom agent tasks whose original verification was a
+ * single `V3Evaluator.ask({question})` YES/NO call. Each criterion becomes
+ * a 1-point rubric item.
+ *
+ * For tasks that already have a concrete predicate ("Does the page show
+ * flights from SF to NY?"), pass the predicate verbatim. For the lazy
+ * "did the agent complete this task successfully? <instruction>" pattern,
+ * pass the instruction.
+ */
+import type { Rubric } from "@browserbasehq/stagehand";
+
+export function adHocRubric(...criteria: string[]): Rubric {
+  if (criteria.length === 0) {
+    throw new Error("adHocRubric requires at least one criterion");
+  }
+  return {
+    items: criteria.map((c) => ({
+      criterion: c,
+      description: c,
+      maxPoints: 1,
+    })),
+  };
+}

packages/evals/framework/adHocRubric.ts

@@ -0,0 +1,163 @@
+import {
+  V3Evaluator,
+  normalizeRubric,
+  type AgentInstance,
+  type AgentExecuteOptions,
+  type AgentResult,
+  type EvaluationResult,
+  type Rubric,
+  type TaskSpec,
+  type Trajectory,
+  type V3,
+} from "@browserbasehq/stagehand";
+
+import { RubricCache } from "./rubricCache.js";
+import { TrajectoryRecorder } from "./trajectoryRecorder.js";
+
+export interface RunWithVerifierOptions {
+  v3: V3;
+  agent: AgentInstance;
+  taskSpec: TaskSpec;
+  /**
+   * Dataset name for rubric cache partitioning. Each task lives under
+   * `.rubric-cache/<dataset>/<task-id>.json`.
+   */
+  dataset: string;
+  /** Agent execute options. `instruction` is filled from taskSpec.instruction. */
+  agentOptions?: Omit<AgentExecuteOptions, "instruction">;
+  /** Override the run id (defaults to ISO timestamp). */
+  runId?: string;
+  /** Override trajectory persistence root. */
+  trajectoryRoot?: string;
+}
+
+export interface RunWithVerifierResult {
+  trajectory: Trajectory;
+  evaluationResult: EvaluationResult;
+  agentResult: AgentResult;
+  /** Resolved rubric (precomputed, cached, or freshly generated). */
+  rubric: Rubric;
+  /** Where the trajectory was persisted (or would have been, if disabled). */
+  trajectoryDir: string;
+}
+
+export async function runWithVerifier(
+  opts: RunWithVerifierOptions,
+): Promise<RunWithVerifierResult> {
+  const { v3, agent, taskSpec, dataset, agentOptions, runId, trajectoryRoot } =
+    opts;
+  const evaluator = new V3Evaluator(v3, { backend: "verifier" });
+
+  // ── Resolve rubric ──────────────────────────────────────────────────────
+  let resolvedRubric: Rubric;
+  if (taskSpec.precomputedRubric) {
+    resolvedRubric = normalizeRubric(taskSpec.precomputedRubric)!;
+  } else if (process.env.VERIFIER_DISABLE_RUBRIC_CACHE === "1") {
+    resolvedRubric = await evaluator.generateRubric(taskSpec);
+  } else {
+    const cache = new RubricCache({ dataset });
+    resolvedRubric = await cache.getOrGenerate(taskSpec, evaluator);
+  }
+
+  // Hand a fully-hydrated TaskSpec to the verifier so it doesn't regenerate.
+  const hydratedTaskSpec: TaskSpec = {
+    ...taskSpec,
+    precomputedRubric: resolvedRubric,
+  };
+
+  // ── Record trajectory around agent.execute() ───────────────────────────
+  const recorder = new TrajectoryRecorder({
+    taskSpec: hydratedTaskSpec,
+    runId,
+    outputRoot: trajectoryRoot,
+  });
+  const { callbacks: userCallbacks, ...restAgentOptions } = agentOptions ?? {};
+
+  let agentResult: AgentResult;
+  let recorderStatus: "complete" | "aborted" | "error" = "complete";
+  try {
+    agentResult = await agent.execute({
+      ...restAgentOptions,
+      instruction: taskSpec.instruction,
+      callbacks: {
+        ...userCallbacks,
+        onEvidence: async (event) => {
+          recorder.record(event);
+          await userCallbacks?.onEvidence?.(event);
+        },
+      },
+    });
+  } catch (e) {
+    recorderStatus = "error";
+    // Re-throw after persisting so the bench task can decide how to report.
+    const wrapped = e instanceof Error ? e : new Error(String(e));
+    try {
+      const trajectory = await recorder.finish({ status: recorderStatus });
+      Object.assign(wrapped, { trajectoryDir: recorder.directory, trajectory });
+    } catch {
+      // Persistence failure must not mask the original agent error.
+    }
+    throw wrapped;
+  }
+
+  const trajectory = await recorder.finish({
+    status: recorderStatus,
+    finalAnswer: agentResult.message,
+    usage: agentResult.usage,
+  });
+
+  // ── Verify ──────────────────────────────────────────────────────────────
+  const evaluationResult = await evaluator.verify(trajectory);
+  await recorder.persistResult(evaluationResult);
+
+  return {
+    trajectory,
+    evaluationResult,
+    agentResult,
+    rubric: resolvedRubric,
+    trajectoryDir: recorder.directory,
+  };
+}
+
+/**
+ * Decide bench task success from an EvaluationResult using the --success flag's
+ * semantics.
+ *
+ * `outcome` (default) — strict binary outcome.
+ * `process`           — rubric process score ≥ threshold (default 0.8).
+ * `both`              — both conditions must hold.
+ */
+export type EvalSuccessMode = "outcome" | "process" | "both";
+
+export function resolveEvalSuccessMode(mode: unknown): EvalSuccessMode {
+  if (typeof mode !== "string") return "outcome";
+  const normalized = mode.trim().toLowerCase();
+  if (
+    normalized === "outcome" ||
+    normalized === "process" ||
+    normalized === "both"
+  ) {
+    return normalized;
+  }
+  return "outcome";
+}
+
+export function evaluationResultToSuccess(
+  result: EvaluationResult,
+  mode: unknown = "outcome",
+  processThreshold = 0.8,
+): boolean {
+  const resolvedMode = resolveEvalSuccessMode(mode);
+  const outcomeOk = result.outcomeSuccess;
+  const processOk =
+    typeof result.processScore === "number" &&
+    result.processScore >= processThreshold;
+  switch (resolvedMode) {
+    case "outcome":
+      return outcomeOk;
+    case "process":
+      return processOk;
+    case "both":
+      return outcomeOk && processOk;
+  }
+}

packages/evals/framework/verifierAdapter.ts

@@ -29,7 +29,7 @@ const JSONL_PATH = path.join(
   "WebTailBench_data.jsonl",
 );
 
-interface Rubric {
+interface RawRubric {
   items: Array<Record<string, unknown>>;
 }
 
@@ -38,7 +38,7 @@ interface LocalRow {
   category?: string;
   ques: string;
   web?: string;
-  precomputed_rubric?: Rubric;
+  precomputed_rubric?: RawRubric;
 }
 
 /**
@@ -114,12 +114,12 @@ async function main(): Promise<void> {
     );
   }
 
-  const rubricsById = new Map<string, Rubric>();
+  const rubricsById = new Map<string, RawRubric>();
   for (let i = 1; i < rows.length; i++) {
     const cols = rows[i];
     if (!cols[idIdx]) continue;
     try {
-      const parsed = JSON.parse(cols[rubricIdx]) as Rubric;
+      const parsed = JSON.parse(cols[rubricIdx]) as RawRubric;
       rubricsById.set(cols[idIdx], parsed);
     } catch (e) {
       console.warn(
@@ -149,7 +149,7 @@ async function main(): Promise<void> {
   }
 
   console.log(
-    `  ✓ matched ${matched}/${inLines.length} rows; ${missing} unmatched (will fall back to Step 0a generation)`,
+    `  ✓ matched ${matched}/${inLines.length} rows; ${missing} unmatched (will fall back to generated rubrics)`,
   );
 
   await fs.writeFile(JSONL_PATH, out.join("\n") + "\n", "utf8");

packages/evals/scripts/backfill-webtailbench-rubrics.ts

@@ -52,7 +52,23 @@ export const buildOnlineMind2WebTestcases = (
   }
 
   const candidates = parseJsonlRows(lines, isMind2WebRow);
-  const rows = applySampling(candidates, sampleCount, maxCases);
+
+  // EVAL_ONLINEMIND2WEB_IDS restricts the suite to exactly those task ids,
+  // preserving the order given and ignoring sampling / limit knobs.
+  const explicitIds = process.env.EVAL_ONLINEMIND2WEB_IDS
+    ? process.env.EVAL_ONLINEMIND2WEB_IDS.split(",")
+        .map((s) => s.trim())
+        .filter(Boolean)
+    : null;
+  let rows: Mind2WebRow[];
+  if (explicitIds && explicitIds.length > 0) {
+    const byId = new Map(candidates.map((r) => [r.task_id, r]));
+    rows = explicitIds
+      .map((id) => byId.get(id))
+      .filter((r): r is Mind2WebRow => Boolean(r));
+  } else {
+    rows = applySampling(candidates, sampleCount, maxCases);
+  }
 
   const allTestcases: Testcase[] = [];
   for (const modelEntry of normalizeAgentModelEntries(models)) {

packages/evals/suites/onlineMind2Web.ts

@@ -1,5 +1,5 @@
 import type { Testcase, EvalInput, AgentModelEntry } from "../types/evals.js";
-import type { AvailableModel } from "@browserbasehq/stagehand";
+import { normalizeRubric, type AvailableModel } from "@browserbasehq/stagehand";
 import { tasksConfig } from "../taskConfig.js";
 import { getPackageRootDir } from "../runtimePaths.js";
 import {
@@ -32,6 +32,12 @@ export const buildWebTailBenchTestcases = (
     ques: string;
     category?: string;
     web?: string;
+    /**
+     * Per-task rubric ported from microsoft/WebTailBench-v1-rubrics.tsv
+     * via packages/evals/scripts/backfill-webtailbench-rubrics.ts.
+     * When present, the verifier uses these upstream criteria directly.
+     */
+    precomputed_rubric?: unknown;
     [key: string]: unknown;
   };
 
@@ -42,7 +48,23 @@ export const buildWebTailBenchTestcases = (
   }
 
   const candidates = parseJsonlRows(lines, isWebTailBenchRow);
-  const rows = applySampling(candidates, sampleCount, maxCases);
+
+  // EVAL_WEBTAILBENCH_IDS restricts the suite to exactly those task IDs,
+  // preserving the order given and ignoring sampling / limit knobs.
+  const explicitIds = process.env.EVAL_WEBTAILBENCH_IDS
+    ? process.env.EVAL_WEBTAILBENCH_IDS.split(",")
+        .map((s) => s.trim())
+        .filter(Boolean)
+    : null;
+  let rows: WebTailBenchRow[];
+  if (explicitIds && explicitIds.length > 0) {
+    const byId = new Map(candidates.map((r) => [r.id, r]));
+    rows = explicitIds
+      .map((id) => byId.get(id))
+      .filter((r): r is WebTailBenchRow => Boolean(r));
+  } else {
+    rows = applySampling(candidates, sampleCount, maxCases);
+  }
 
   const allTestcases: Testcase[] = [];
   for (const modelEntry of normalizeAgentModelEntries(models)) {
@@ -57,6 +79,7 @@ export const buildWebTailBenchTestcases = (
           category: row.category,
           ques: row.ques,
           web: row.web,
+          precomputed_rubric: normalizeRubric(row.precomputed_rubric),
         },
       };
       const taskCategories =