|
| 1 | +/** |
| 2 | + * intelligence-coding-bench — the WebCode harness×model coding benchmark (see ../webcode-matrix), |
| 3 | + * instrumented with the FULL Tangle Intelligence SDK. It reuses the EXACT benchmark next door — the same |
| 4 | + * harness×model grid and the same post-Aug-2025 WebCode tasks graded by hidden tests — and adds nothing |
| 5 | + * but the intelligence wiring. Per harness×model you get: did it pass, what it cost, the per-tool |
| 6 | + * waterfall, and the spans streamed to your trace collector. |
| 7 | + * |
| 8 | + * Three intelligence layers, all on one cell: |
| 9 | + * 1. BOUNDARY (the bill + the control) — `withTangleIntelligence(cell, { project, effort })`. |
| 10 | + * `effort ∈ off | eco | standard | thorough | max`; `'off'` is the PROVABLE passthrough floor — |
| 11 | + * intelligence spend clamped to 0, the cell still runs. This is the one knob that gates spend. |
| 12 | + * 2. WATERFALL (the cost truth) — `createWaterfallCollector()` on the run. The sum of its spans IS the |
| 13 | + * billed run cost, per tool/phase — no separate tally to drift. |
| 14 | + * 3. OTLP (the production trace pipe) — `createOtelExporter()` + `loopEventToOtelSpan`. Streams every |
| 15 | + * span to your OTLP/HTTP collector (set `OTEL_EXPORTER_OTLP_ENDPOINT`); a no-op when unset. |
| 16 | + * |
| 17 | + * The intelligence attaches at TWO seams: the BOUNDARY wraps the whole cell (`withTangleIntelligence` |
| 18 | + * works over any async fn), and the INTERNAL trace rides `openSandboxRun`'s `hooks` (the only run verb |
| 19 | + * here that emits per-tool spans). Same pattern instruments `runProfileMatrix`'s dispatch wholesale. |
| 20 | + * |
| 21 | + * Run: |
| 22 | + * TANGLE_API_KEY=… SANDBOX_API_KEY=… EXA_API_KEY=… [EFFORT=standard] [OTEL_EXPORTER_OTLP_ENDPOINT=…] \ |
| 23 | + * tsx examples/intelligence-coding-bench/intelligence-coding-bench.ts |
| 24 | + */ |
| 25 | + |
| 26 | +import type { AgentProfile } from '@tangle-network/agent-interface' |
| 27 | +import { |
| 28 | + composeRuntimeHooks, |
| 29 | + createOtelExporter, |
| 30 | + loopEventToOtelSpan, |
| 31 | + type RuntimeHooks, |
| 32 | +} from '@tangle-network/agent-runtime' |
| 33 | +import { type EffortTier, withTangleIntelligence } from '@tangle-network/agent-runtime/intelligence' |
| 34 | +import { |
| 35 | + type AgentRunSpec, |
| 36 | + createWaterfallCollector, |
| 37 | + openSandboxRun, |
| 38 | + type SandboxClient, |
| 39 | +} from '@tangle-network/agent-runtime/loops' |
| 40 | +import type { BackendType } from '@tangle-network/sandbox' |
| 41 | +import { |
| 42 | + type WebCodeTask, |
| 43 | + profiles as webcodeGrid, |
| 44 | + tasks as webcodeTasks, |
| 45 | +} from '../webcode-matrix/webcode-matrix' |
| 46 | + |
| 47 | +const routerBaseUrl = process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1' |
| 48 | +const effort = (process.env.EFFORT ?? 'standard') as EffortTier |
| 49 | +const project = process.env.TANGLE_PROJECT ?? 'webcode-bench' |
| 50 | + |
| 51 | +interface CellInput { |
| 52 | + profile: AgentProfile |
| 53 | + task: WebCodeTask |
| 54 | +} |
| 55 | +interface CellResult { |
| 56 | + passed: boolean |
| 57 | + usd: number |
| 58 | + ms: number |
| 59 | + waterfall: string |
| 60 | +} |
| 61 | + |
| 62 | +/** One instrumented cell: run a harness×model on a WebCode task in its own sandbox with the INTERNAL trace |
| 63 | + * collected (cost waterfall) AND streamed (OTLP), then score on the hidden tests. Wrapping this in |
| 64 | + * `withTangleIntelligence` (below) adds the BOUNDARY layer. */ |
| 65 | +function instrumentedCell(client: SandboxClient): (input: CellInput) => Promise<CellResult> { |
| 66 | + return async ({ profile, task }) => { |
| 67 | + const harness = String(profile.metadata?.harness ?? 'opencode') |
| 68 | + const model = String(profile.metadata?.model ?? '') |
| 69 | + |
| 70 | + // LAYER 2 + 3 — the INTERNAL trace: the per-tool waterfall (cost) AND OTLP spans, both as run hooks. |
| 71 | + const waterfall = createWaterfallCollector() |
| 72 | + const otel = createOtelExporter() // undefined unless OTEL_EXPORTER_OTLP_ENDPOINT (or config) is set |
| 73 | + const otelHook: RuntimeHooks = otel |
| 74 | + ? { |
| 75 | + onEvent: (e) => { |
| 76 | + otel.exportSpan( |
| 77 | + loopEventToOtelSpan( |
| 78 | + { |
| 79 | + kind: e.target, |
| 80 | + runId: e.runId, |
| 81 | + timestamp: e.timestamp, |
| 82 | + payload: (e.payload ?? {}) as object, |
| 83 | + }, |
| 84 | + e.runId, |
| 85 | + ), |
| 86 | + ) |
| 87 | + }, |
| 88 | + } |
| 89 | + : {} |
| 90 | + |
| 91 | + const agentRun: AgentRunSpec<string> = { |
| 92 | + profile, |
| 93 | + name: profile.name ?? harness, |
| 94 | + taskToPrompt: (t) => t, |
| 95 | + sandboxOverrides: { |
| 96 | + // Only the WebCode search creds — never router/model credentials into the box. |
| 97 | + env: { |
| 98 | + TANGLE_SEARCH_DEFAULT_PROVIDER: 'exa', |
| 99 | + ...(process.env.EXA_API_KEY ? { EXA_API_KEY: process.env.EXA_API_KEY } : {}), |
| 100 | + }, |
| 101 | + backend: { |
| 102 | + type: harness as BackendType, |
| 103 | + model: { provider: 'openai-compat', model, baseUrl: routerBaseUrl }, |
| 104 | + }, |
| 105 | + }, |
| 106 | + } |
| 107 | + |
| 108 | + const run = await openSandboxRun<{ passed: boolean }>( |
| 109 | + client, |
| 110 | + { |
| 111 | + agentRun, |
| 112 | + scenarioId: task.id, |
| 113 | + signal: new AbortController().signal, |
| 114 | + hooks: composeRuntimeHooks(waterfall.hooks, otelHook), |
| 115 | + }, |
| 116 | + { kind: 'events', fromEvents: () => ({ passed: false }) }, |
| 117 | + ) |
| 118 | + await run.start(task.prompt) |
| 119 | + const res = await run.box.exec?.(task.testCmd) |
| 120 | + await otel?.flush() |
| 121 | + |
| 122 | + const report = waterfall.report() |
| 123 | + return { |
| 124 | + passed: (res?.exitCode ?? 1) === 0, |
| 125 | + usd: report.totalUsd, |
| 126 | + ms: report.totalMs, |
| 127 | + waterfall: waterfall.render({ maxRows: 8 }), |
| 128 | + } |
| 129 | + } |
| 130 | +} |
| 131 | + |
| 132 | +/** Run the WebCode grid × tasks with the full intelligence stack on every cell. */ |
| 133 | +export async function runIntelligenceCodingBench(client: SandboxClient): Promise<void> { |
| 134 | + // LAYER 1 — the BOUNDARY: every cell runs under `withTangleIntelligence` — traced + billed, effort-gated. |
| 135 | + // `effort: 'off'` clamps intelligence spend to 0 (the provable passthrough floor) while still running. |
| 136 | + const smartCell = withTangleIntelligence(instrumentedCell(client), { project, effort }) |
| 137 | + |
| 138 | + console.log(`intelligence-coding-bench · effort=${effort} · project=${project}`) |
| 139 | + console.log(`${'harness·model'.padEnd(30)}${'task'.padEnd(14)}result cost wall\n`) |
| 140 | + let shownWaterfall = false |
| 141 | + for (const profile of webcodeGrid) { |
| 142 | + for (const task of webcodeTasks) { |
| 143 | + const r = await smartCell({ profile, task }) |
| 144 | + console.log( |
| 145 | + `${(profile.name ?? '').padEnd(30)}${task.id.padEnd(14)}${r.passed ? 'PASS' : 'fail'} $${r.usd.toFixed(4)} ${(r.ms / 1000).toFixed(1)}s`, |
| 146 | + ) |
| 147 | + // Show the per-tool cost waterfall (layer 2) once — the same spans the $ column sums. |
| 148 | + if (!shownWaterfall) { |
| 149 | + console.log(`\n — per-tool cost waterfall (layer 2), one cell —\n${r.waterfall}\n`) |
| 150 | + shownWaterfall = true |
| 151 | + } |
| 152 | + } |
| 153 | + } |
| 154 | +} |
| 155 | + |
| 156 | +// Run it live — mirrors ../webcode-matrix's client wiring. |
| 157 | +if (import.meta.url === `file://${process.argv[1]}`) { |
| 158 | + const { SandboxClient } = (await import('@tangle-network/sandbox')) as { |
| 159 | + SandboxClient: new (o: { apiKey: string; baseUrl: string }) => SandboxClient |
| 160 | + } |
| 161 | + const apiKey = process.env.SANDBOX_API_KEY |
| 162 | + if (!apiKey) throw new Error('SANDBOX_API_KEY required') |
| 163 | + const client = new SandboxClient({ |
| 164 | + apiKey, |
| 165 | + baseUrl: process.env.SANDBOX_BASE_URL ?? 'https://sandbox.tangle.tools', |
| 166 | + }) |
| 167 | + await runIntelligenceCodingBench(client) |
| 168 | +} |
0 commit comments