feat: cost tracking, percentiles, badges, compare + Opus 4.7

- Real token capture: agent emits USAGE line; parseUsageLine parses it
- pricing.ts maps model→$/MTok (Anthropic list prices); cost flows to summary
- p50/p95 plumbing via linear interpolation; surfaces when runs>1
- Badge endpoint /badge/<slug>.svg?metric=task|pass|score (shields-style SVG)
- Compare page /compare?a=x&b=y with scoreboard + Δ time + cost ratio
- Logo + favicon (Hunter Reticle mark) integrated via nav + metadata
- URL params for ranking sort, Reproduce Now CTA, criteria matrix on task pages
- Add Opus 4.7 to agent roster (released yesterday — trending)

Rankings (10 tasks, LLM-as-judge, real tokens + cost):
  Opus 4.7   10/10  8.4s  $0.56   ← fastest perfect scorer
  Opus 4.6   10/10  9.8s  $0.44
  Sonnet 4.6 10/10  9.8s  $0.11   ← 4× cheaper than Opus
  Haiku 4.5   8/10  4.6s  $0.03

Tests 98/98. Lint + build green.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: iamzifei

.gitignore

@@ -143,3 +143,12 @@ vite.config.ts.timestamp-*
 
 # macOS
 .DS_Store
+.vercel
+
+# Playwright/browser snapshots (ephemeral)
+.playwright-mcp/
+# Ad-hoc local screenshots & snapshots
+homepage_screenshot.png
+homepage_snapshot.md
+# Ad-hoc eval workspaces
+test-run/

packages/agent-eval/src/eval/pricing.ts

@@ -0,0 +1,51 @@
+/**
+ * Model pricing — USD per million tokens.
+ *
+ * Numbers track Anthropic's published list prices as of 2026. Keep the map
+ * in sync with api.anthropic.com/pricing when adding new models; the eval
+ * engine falls back to 0 for unknown model IDs rather than guessing, since
+ * an incorrect price is worse than no price at all.
+ */
+
+export interface ModelPrice {
+  /** USD per million input tokens */
+  inputPerMTok: number;
+  /** USD per million output tokens */
+  outputPerMTok: number;
+}
+
+const PRICES: Record<string, ModelPrice> = {
+  // Anthropic Claude 4.x family
+  "claude-opus-4-6": { inputPerMTok: 15, outputPerMTok: 75 },
+  "claude-opus-4-7": { inputPerMTok: 15, outputPerMTok: 75 },
+  "claude-sonnet-4-6": { inputPerMTok: 3, outputPerMTok: 15 },
+  "claude-haiku-4-5": { inputPerMTok: 1, outputPerMTok: 5 },
+  "claude-haiku-4-5-20251001": { inputPerMTok: 1, outputPerMTok: 5 },
+  // Short aliases — agent scripts frequently refer to these
+  opus: { inputPerMTok: 15, outputPerMTok: 75 },
+  sonnet: { inputPerMTok: 3, outputPerMTok: 15 },
+  haiku: { inputPerMTok: 1, outputPerMTok: 5 },
+};
+
+/**
+ * Compute the USD cost of a single run given model ID and token counts.
+ * Returns 0 when the model is unknown so callers can safely render it.
+ */
+export function computeCostUsd(
+  model: string | undefined,
+  inputTokens: number | undefined,
+  outputTokens: number | undefined,
+): number {
+  if (!model || inputTokens == null || outputTokens == null) return 0;
+  const price = PRICES[model] ?? PRICES[model.toLowerCase()];
+  if (!price) return 0;
+  return (
+    (inputTokens * price.inputPerMTok) / 1_000_000 +
+    (outputTokens * price.outputPerMTok) / 1_000_000
+  );
+}
+
+/** Expose the price map so tests and tooling can enumerate supported models. */
+export function getPrice(model: string): ModelPrice | null {
+  return PRICES[model] ?? PRICES[model.toLowerCase()] ?? null;
+}

packages/agent-eval/src/eval/task-eval.ts

@@ -22,6 +22,7 @@ import {
 import Anthropic from "@anthropic-ai/sdk";
 import type { TaskEvalConfig } from "../config/task-schema.js";
 import { callWithRetry } from "./llm-client.js";
+import { computeCostUsd } from "./pricing.js";
 
 /** Result of evaluating a single success criterion */
 export interface CriterionResult {
@@ -40,6 +41,12 @@ export interface TaskRunResult {
   output: string;
   /** Estimated token count from agent output length (rough proxy) */
   estimatedTokens: number;
+  /** Real input tokens reported by the agent via `USAGE: input=X output=Y` (if emitted) */
+  inputTokens?: number;
+  /** Real output tokens reported by the agent (if emitted) */
+  outputTokens?: number;
+  /** Computed USD cost for this run, if pricing + real tokens are available */
+  costUsd?: number;
 }
 
 /** Aggregated result across all runs */
@@ -49,7 +56,13 @@ export interface TaskEvalResult {
   runs: TaskRunResult[];
   successRate: number;
   avgDurationMs: number;
+  /** 50th percentile duration across runs — equal to avg when runs=1 */
+  p50DurationMs: number;
+  /** 95th percentile duration — useful once runs>=3 to reveal tail latency */
+  p95DurationMs: number;
   avgTokens: number;
+  /** Average USD cost across runs — 0 if cost isn't known */
+  avgCostUsd: number;
   totalRuns: number;
   totalPassed: number;
 }
@@ -168,9 +181,20 @@ export async function runTaskEvaluation(options: {
 
     const allPassed = criteriaResults.every((c) => c.passed);
 
-    // Rough token estimate: ~4 chars per token
+    // Rough token estimate: ~4 chars per token. Used as a fallback when the
+    // agent doesn't report real token usage on its USAGE line.
     const estimatedTokens = Math.round(output.length / 4);
 
+    // Parse `USAGE: input=<n> output=<m> [model=<id>]` emitted by agent scripts
+    // so we can track real token counts and compute accurate cost.
+    const usage = parseUsageLine(output);
+
+    const costUsd = computeCostUsd(
+      usage?.model,
+      usage?.inputTokens,
+      usage?.outputTokens,
+    );
+
     runs.push({
       runIndex: i + 1,
       success: allPassed,
@@ -179,34 +203,87 @@ export async function runTaskEvaluation(options: {
       exitCode,
       output: output.slice(0, 5000), // Truncate for storage
       estimatedTokens,
+      inputTokens: usage?.inputTokens,
+      outputTokens: usage?.outputTokens,
+      costUsd: costUsd > 0 ? costUsd : undefined,
     });
   }
 
   // Aggregate
   const totalPassed = runs.filter((r) => r.success).length;
+  const durations = runs.map((r) => r.durationMs);
   const avgDurationMs =
-    runs.length > 0
-      ? Math.round(runs.reduce((s, r) => s + r.durationMs, 0) / runs.length)
+    durations.length > 0
+      ? Math.round(durations.reduce((s, d) => s + d, 0) / durations.length)
       : 0;
   const avgTokens =
     runs.length > 0
       ? Math.round(
           runs.reduce((s, r) => s + r.estimatedTokens, 0) / runs.length,
         )
       : 0;
+  const runsWithCost = runs.filter((r) => r.costUsd != null);
+  const avgCostUsd =
+    runsWithCost.length > 0
+      ? runsWithCost.reduce((s, r) => s + (r.costUsd ?? 0), 0) /
+        runsWithCost.length
+      : 0;
 
   return {
     taskName: config.task.name,
     agentCommand: `${config.agent.command} ${config.agent.args.join(" ")}`,
     runs,
     successRate: runs.length > 0 ? totalPassed / runs.length : 0,
     avgDurationMs,
+    p50DurationMs: percentile(durations, 50),
+    p95DurationMs: percentile(durations, 95),
     avgTokens,
+    avgCostUsd,
     totalRuns: runs.length,
     totalPassed,
   };
 }
 
+/**
+ * Compute the requested percentile of a numeric array. Uses linear
+ * interpolation between adjacent ranks (matches numpy's default), which
+ * gives sensible results even for small sample sizes.
+ */
+export function percentile(values: number[], p: number): number {
+  if (values.length === 0) return 0;
+  if (values.length === 1) return Math.round(values[0] ?? 0);
+  const sorted = [...values].sort((a, b) => a - b);
+  const rank = (p / 100) * (sorted.length - 1);
+  const lo = Math.floor(rank);
+  const hi = Math.ceil(rank);
+  const weight = rank - lo;
+  const value = (sorted[lo] ?? 0) * (1 - weight) + (sorted[hi] ?? 0) * weight;
+  return Math.round(value);
+}
+
+/**
+ * Parse the `USAGE:` marker line that agent scripts emit after calling the
+ * model API. Accepts formats like:
+ *   USAGE: input=1234 output=567
+ *   USAGE: input=1234 output=567 model=claude-sonnet-4-6
+ * Returns null if no marker is present so callers fall back to rough estimates.
+ */
+export function parseUsageLine(output: string): {
+  inputTokens: number;
+  outputTokens: number;
+  model?: string;
+} | null {
+  const match = output.match(
+    /^USAGE:\s+input=(\d+)\s+output=(\d+)(?:\s+model=([\w.\-:]+))?/m,
+  );
+  if (!match) return null;
+  return {
+    inputTokens: Number.parseInt(match[1] ?? "0", 10),
+    outputTokens: Number.parseInt(match[2] ?? "0", 10),
+    model: match[3],
+  };
+}
+
 /**
  * Use LLM-as-judge to evaluate each success criterion against the agent output.
  */

packages/agent-eval/tests/pricing.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vitest";
+import { computeCostUsd, getPrice } from "../src/eval/pricing.js";
+
+describe("pricing", () => {
+  it("returns 0 for unknown models so cost renders safely", () => {
+    expect(computeCostUsd("unknown-model", 1000, 1000)).toBe(0);
+  });
+
+  it("returns 0 when token counts are missing", () => {
+    expect(computeCostUsd("claude-sonnet-4-6", undefined, 100)).toBe(0);
+    expect(computeCostUsd("claude-sonnet-4-6", 100, undefined)).toBe(0);
+    expect(computeCostUsd(undefined, 100, 100)).toBe(0);
+  });
+
+  it("computes Sonnet 4.6 cost: 1M input + 1M output = $3 + $15 = $18", () => {
+    const cost = computeCostUsd("claude-sonnet-4-6", 1_000_000, 1_000_000);
+    expect(cost).toBeCloseTo(18, 5);
+  });
+
+  it("computes Opus 4.6 cost: 1M input + 1M output = $15 + $75 = $90", () => {
+    const cost = computeCostUsd("claude-opus-4-6", 1_000_000, 1_000_000);
+    expect(cost).toBeCloseTo(90, 5);
+  });
+
+  it("computes Haiku 4.5 cost: 1M input + 1M output = $1 + $5 = $6", () => {
+    const cost = computeCostUsd("claude-haiku-4-5", 1_000_000, 1_000_000);
+    expect(cost).toBeCloseTo(6, 5);
+  });
+
+  it("accepts short aliases like 'sonnet' for ergonomics", () => {
+    const cost = computeCostUsd("sonnet", 1_000_000, 1_000_000);
+    expect(cost).toBeCloseTo(18, 5);
+  });
+
+  it("getPrice returns null for unknown models", () => {
+    expect(getPrice("gpt-9000")).toBeNull();
+  });
+
+  it("getPrice returns structured data for known models", () => {
+    expect(getPrice("claude-sonnet-4-6")).toEqual({
+      inputPerMTok: 3,
+      outputPerMTok: 15,
+    });
+  });
+});

packages/agent-eval/tests/task-eval.test.ts

@@ -28,7 +28,9 @@ vi.mock("@anthropic-ai/sdk", () => {
   return { default: MockAnthropic };
 });
 
-const { runTaskEvaluation } = await import("../src/eval/task-eval.js");
+const { runTaskEvaluation, percentile, parseUsageLine } = await import(
+  "../src/eval/task-eval.js"
+);
 
 describe("runTaskEvaluation", () => {
   it("should evaluate a simple CLI agent task", async () => {
@@ -128,4 +130,75 @@ describe("runTaskEvaluation", () => {
     // The echo output should contain the substituted description
     expect(result.runs[0]?.output).toContain("Create a greeting");
   });
+
+  it("populates p50/p95 duration fields", async () => {
+    const result = await runTaskEvaluation({
+      config: {
+        task: {
+          name: "Percentile test",
+          description: "Checks duration percentiles exist",
+          success_criteria: ["Task completed"],
+        },
+        agent: {
+          type: "cli",
+          command: "echo",
+          args: ["done"],
+        },
+        eval: { timeout: 10, runs: 3 },
+      },
+      apiKey: "test-key",
+    });
+
+    expect(result.p50DurationMs).toBeGreaterThanOrEqual(0);
+    expect(result.p95DurationMs).toBeGreaterThanOrEqual(result.p50DurationMs);
+    // avgCostUsd is 0 when the agent doesn't emit a USAGE line (like `echo`)
+    expect(result.avgCostUsd).toBe(0);
+  });
+});
+
+describe("percentile", () => {
+  it("returns 0 for empty arrays", () => {
+    expect(percentile([], 50)).toBe(0);
+  });
+
+  it("returns the single value when only one sample", () => {
+    expect(percentile([42], 95)).toBe(42);
+  });
+
+  it("computes p50 of [1,2,3,4,5] as 3", () => {
+    expect(percentile([1, 2, 3, 4, 5], 50)).toBe(3);
+  });
+
+  it("interpolates p95 of [10,20,30,40,50] near the top", () => {
+    // rank = 0.95 * 4 = 3.8 → interpolates between 40 and 50 → 48
+    expect(percentile([10, 20, 30, 40, 50], 95)).toBe(48);
+  });
+
+  it("handles unsorted input", () => {
+    expect(percentile([5, 1, 4, 2, 3], 50)).toBe(3);
+  });
+});
+
+describe("parseUsageLine", () => {
+  it("returns null when no USAGE line is present", () => {
+    expect(parseUsageLine("Task completed.\nCreated: foo.js")).toBeNull();
+  });
+
+  it("parses input/output tokens", () => {
+    const parsed = parseUsageLine(
+      "Created: foo.js\nUSAGE: input=1234 output=567",
+    );
+    expect(parsed).toEqual({ inputTokens: 1234, outputTokens: 567 });
+  });
+
+  it("parses input/output tokens + model", () => {
+    const parsed = parseUsageLine(
+      "Created: foo.js\nUSAGE: input=1234 output=567 model=claude-sonnet-4-6",
+    );
+    expect(parsed).toEqual({
+      inputTokens: 1234,
+      outputTokens: 567,
+      model: "claude-sonnet-4-6",
+    });
+  });
 });