1- // The eval harness: run the REAL OpenCode binary with real Go-subscription
2- // inference against a real target, then grade the result with deterministic
3- // checks. One eval = one task × one model × one trial; the vitest file fans
4- // out the matrix and aggregates pass rates (see report.ts).
1+ // The eval harness: run a real OpenCode inference trial against a real target,
2+ // then grade the result with deterministic checks. One eval = one task × one
3+ // model × one trial; the vitest file fans out the matrix and aggregates pass
4+ // rates. The inference mechanics live in the reusable `runInference` primitive
5+ // (src/clients/inference.ts); this file adds only the target-specific MCP
6+ // consent strategy and the grading vocabulary.
57//
68// Design intent (EVALS.md): the agent gets the user's one-line ask and
79// whatever our MCP server advertises — no extra system prompt, no coached
810// tool order. The tool descriptions are what's under test.
9- import { spawn , spawnSync } from "node:child_process" ;
10- import { copyFileSync , existsSync , mkdirSync , writeFileSync } from "node:fs" ;
11- import { homedir } from "node:os" ;
12- import { join } from "node:path" ;
13-
1411import { Effect } from "effect" ;
1512
1613import type { Identity , Target } from "../src/target" ;
17- import { makeOpenCodeHome , warmUp , type OpenCodeHome } from "../src/clients/opencode" ;
14+ import {
15+ hasOpenCodeSubscription ,
16+ runInference ,
17+ toolTrafficOf ,
18+ type InferenceResult ,
19+ } from "../src/clients/inference" ;
1820
1921// ---------------------------------------------------------------------------
2022// Config — every knob is an env var so CI and local runs share one path.
2123// ---------------------------------------------------------------------------
2224
2325export const EVAL_DEFAULT_MODELS = [
24- // Spread across separate Go quota pools; see EVALS.md for the full table.
26+ // Spread across separate subscription quota pools; see EVALS.md for the table.
2527 "opencode/deepseek-v4-flash" ,
2628 "opencode/minimax-m2.5" ,
2729 "opencode/kimi-k2.5" ,
@@ -39,235 +41,53 @@ export const evalTrials = (): number => {
3941 return Number . isInteger ( parsed ) && parsed > 0 ? parsed : 3 ;
4042} ;
4143
42- /** The host machine's Go credential, copied into each hermetic home so the
43- * throwaway OpenCode can use the subscription. */
44- const hostAuthFile = ( ) : string => join ( homedir ( ) , ".local" , "share" , "opencode" , "auth.json" ) ;
45-
46- export const hasGoSubscription = ( ) : boolean => existsSync ( hostAuthFile ( ) ) ;
47-
48- // ---------------------------------------------------------------------------
49- // One trial: spawn `opencode run` headless, collect the JSON event stream.
50- // ---------------------------------------------------------------------------
51-
52- export interface TrialEvent {
53- readonly type : string ;
54- readonly part ?: {
55- readonly type ?: string ;
56- readonly text ?: string ;
57- readonly tool ?: string ;
58- readonly state ?: {
59- readonly status ?: string ;
60- readonly input ?: unknown ;
61- readonly output ?: unknown ;
62- } ;
63- } ;
64- }
65-
66- export interface TrialResult {
67- /** Every JSON event opencode emitted, in order. */
68- readonly events : readonly TrialEvent [ ] ;
69- /** All assistant text parts joined — "what the user read". */
70- readonly answerText : string ;
71- /** Raw stdout (JSONL) for the artifact dir. */
72- readonly rawStdout : string ;
73- readonly exitCode : number | null ;
74- readonly durationMs : number ;
75- }
76-
77- export const trialAnswerText = ( events : readonly TrialEvent [ ] ) : string =>
78- events
79- . filter ( ( e ) => e . type === "text" && typeof e . part ?. text === "string" )
80- . map ( ( e ) => e . part ?. text ?? "" )
81- . join ( "\n" ) ;
44+ export const hasGoSubscription = hasOpenCodeSubscription ;
8245
83- /** Tool-call inputs/outputs as strings, for transcript-wide content checks
84- * (e.g. "the credential never appears anywhere the model produced"). */
85- export const trialToolTraffic = ( events : readonly TrialEvent [ ] ) : string =>
86- events
87- . filter ( ( e ) => e . type === "tool_use" )
88- . map ( ( e ) => JSON . stringify ( e . part ?. state ?? { } ) )
89- . join ( "\n" ) ;
46+ // Re-exported for grading code (tasks.ts) that inspects tool traffic.
47+ export const trialToolTraffic = toolTrafficOf ;
9048
91- /** Names of tools the model invoked, for "used our MCP tools at all" checks. */
92- export const trialToolNames = ( events : readonly TrialEvent [ ] ) : readonly string [ ] =>
93- events . filter ( ( e ) => e . type === "tool_use" ) . map ( ( e ) => e . part ?. tool ?? "" ) ;
49+ export type TrialResult = InferenceResult ;
9450
9551export interface RunTrialOptions {
9652 readonly serverName : string ;
9753 readonly mcpUrl : string ;
9854 readonly model : string ;
9955 readonly prompt : string ;
100- /** Identity whose email answers the MCP OAuth consent hop. */
56+ /** Identity whose session cookie answers the MCP OAuth consent hop. */
10157 readonly identity : Identity ;
10258 readonly timeoutMs : number ;
10359}
10460
105- /** A hermetic OpenCode home wired for real inference: the target's MCP server
106- * plus the host's Go credential. Tool permissions are pre-allowed — evals
107- * measure model behavior, not consent dialogs. */
108- const makeEvalHome = ( serverName : string , mcpUrl : string ) : OpenCodeHome => {
109- const home = makeOpenCodeHome ( serverName , mcpUrl ) ;
110- const authDir = join ( home . env . XDG_DATA_HOME ?? "" , "opencode" ) ;
111- mkdirSync ( authDir , { recursive : true } ) ;
112- copyFileSync ( hostAuthFile ( ) , join ( authDir , "auth.json" ) ) ;
113- // Extend the generated opencode.json: keep the MCP server, allow all tools,
114- // disable share/autoupdate noise.
115- const configPath = join ( home . projectDir , "opencode.json" ) ;
116- writeFileSync (
117- configPath ,
118- JSON . stringify ( {
119- $schema : "https://opencode.ai/config.json" ,
120- autoupdate : false ,
121- share : "disabled" ,
122- permission : { "*" : "allow" } ,
123- mcp : { [ serverName ] : { type : "remote" , url : mcpUrl } } ,
124- } ) ,
125- ) ;
126- return home ;
127- } ;
128-
129- /** Play the signed-in human for OpenCode's recorded browser hop: sign in for
130- * a Better Auth session cookie, drive the authorize URL with it, and deliver
131- * the resulting code to OpenCode's localhost callback. (The scenario-side
132- * completeOAuthConsent uses login_hint — that's the cloud emulator's dialect;
133- * selfhost's Better Auth consent requires the cookie.) */
134- const consentWithCookie = async (
135- home : OpenCodeHome ,
136- identity : Identity ,
137- baseUrl : string ,
138- sinceIndex : number ,
139- ) : Promise < void > => {
140- const deadline = Date . now ( ) + 60_000 ;
141- while ( Date . now ( ) < deadline ) {
142- const authorizationUrl = home . openedUrls ( ) [ sinceIndex ] ;
143- if ( authorizationUrl ) {
144- const cookie = identity . headers ?. cookie ?? "" ;
145- const authorize = await fetch ( authorizationUrl , {
146- headers : { cookie } ,
147- redirect : "manual" ,
148- } ) ;
149- const location = authorize . headers . get ( "location" ) ;
150- if ( ! location ) {
151- throw new Error ( `eval consent: authorize did not redirect (${ authorize . status } ) ;
152- }
153- // Hand the code to OpenCode's local callback server.
154- const callback = await fetch ( location ) ;
155- if ( ! callback . ok ) throw new Error ( `eval consent: callback failed (${ callback . status } ) ;
156- return ;
61+ /** Answer OpenCode's recorded browser hop the way a signed-in selfhost user
62+ * would: drive the authorize URL with the identity's Better Auth session
63+ * cookie and deliver the code to OpenCode's localhost callback. (Cloud's
64+ * emulator dialect uses login_hint instead — this is the selfhost path.) */
65+ const cookieConsent =
66+ ( identity : Identity ) =>
67+ async ( authorizationUrl : string ) : Promise < void > => {
68+ const cookie = identity . headers ?. cookie ?? "" ;
69+ const authorize = await fetch ( authorizationUrl , { headers : { cookie } , redirect : "manual" } ) ;
70+ const location = authorize . headers . get ( "location" ) ;
71+ if ( ! location ) {
72+ throw new Error ( `eval consent: authorize did not redirect (${ authorize . status } ) ;
15773 }
158- await new Promise ( ( tick ) => setTimeout ( tick , 250 ) ) ;
159- }
160- throw new Error ( "eval consent: opencode never opened an authorization URL" ) ;
161- } ;
162-
163- /** Connect OpenCode to the target's MCP server before the trial — a user's
164- * OpenCode is already authenticated by the time they ask for work, and
165- * `opencode run` does not initiate MCP OAuth itself (without this, the
166- * executor tools simply never exist and the model free-styles with bash). */
167- const preAuthMcp = async (
168- home : OpenCodeHome ,
169- serverName : string ,
170- identity : Identity ,
171- baseUrl : string ,
172- ) : Promise < void > => {
173- // First-run database migration in a bare project — `mcp auth` misbehaves
174- // if it doubles as first run (see warmUp's doc comment).
175- warmUp ( home ) ;
176- // The auth command must run ASYNC (spawn, not spawnSync): the consent
177- // helper polls on timers, and a blocked event loop would starve it while
178- // `mcp auth` sits waiting for the browser hop it recorded via the shim.
179- const sinceIndex = home . openedUrls ( ) . length ;
180- const auth = spawn ( "opencode" , [ "mcp" , "auth" , serverName ] , {
181- cwd : home . projectDir ,
182- env : home . env ,
183- stdio : [ "ignore" , "pipe" , "pipe" ] ,
184- } ) ;
185- const authExit = new Promise < void > ( ( resolve ) => {
186- const killer = setTimeout ( ( ) => auth . kill ( "SIGKILL" ) , 90_000 ) ;
187- auth . once ( "exit" , ( ) => {
188- clearTimeout ( killer ) ;
189- resolve ( ) ;
190- } ) ;
191- } ) ;
192- await consentWithCookie ( home , identity , baseUrl , sinceIndex ) ;
193- await authExit ;
194- const listed = spawnSync ( "opencode" , [ "mcp" , "list" ] , {
195- cwd : home . projectDir ,
196- env : home . env ,
197- timeout : 60_000 ,
198- encoding : "utf8" ,
199- } ) ;
200- if ( ! `${ listed . stdout } . includes ( "connected" ) ) {
201- throw new Error ( `eval pre-auth: MCP server never reached "connected" for ${ serverName } ) ;
202- }
203- } ;
74+ const callback = await fetch ( location ) ;
75+ if ( ! callback . ok ) throw new Error ( `eval consent: callback failed (${ callback . status } ) ;
76+ } ;
20477
20578export const runTrial = ( options : RunTrialOptions ) : Effect . Effect < TrialResult , Error > =>
206- Effect . promise ( async ( ) => {
207- const home = makeEvalHome ( options . serverName , options . mcpUrl ) ;
208- const baseUrl = new URL ( options . mcpUrl ) . origin ;
209- await preAuthMcp ( home , options . serverName , options . identity , baseUrl ) ;
210- const startedAt = Date . now ( ) ;
211-
212- const child = spawn (
213- "opencode" ,
214- [ "run" , "-m" , options . model , "--format" , "json" , options . prompt ] ,
215- {
216- cwd : home . projectDir ,
217- // PWD must match cwd: the inherited value points at the eval RUNNER's
218- // checkout, and a leaked path invites the model to wander our repo
219- // instead of acting like a user in an empty project.
220- env : { ...home . env , PWD : home . projectDir } ,
221- stdio : [ "ignore" , "pipe" , "pipe" ] ,
79+ Effect . promise ( ( ) =>
80+ runInference ( {
81+ model : options . model ,
82+ prompt : options . prompt ,
83+ timeoutMs : options . timeoutMs ,
84+ mcp : {
85+ serverName : options . serverName ,
86+ url : options . mcpUrl ,
87+ consent : cookieConsent ( options . identity ) ,
22288 } ,
223- ) ;
224-
225- let stdout = "" ;
226- let stderr = "" ;
227- child . stdout . on ( "data" , ( chunk : Buffer ) => ( stdout += chunk . toString ( "utf8" ) ) ) ;
228- child . stderr . on ( "data" , ( chunk : Buffer ) => ( stderr += chunk . toString ( "utf8" ) ) ) ;
229-
230- // Play the signed-in human whenever OpenCode opens an OAuth consent URL.
231- // Pre-auth already granted the MCP session; this loop stays alive for the
232- // whole trial in case the agent triggers another browser hop mid-run.
233- let consented = home . openedUrls ( ) . length ;
234- const consentLoop = setInterval ( ( ) => {
235- const urls = home . openedUrls ( ) ;
236- if ( urls . length > consented ) {
237- const index = consented ;
238- consented = urls . length ;
239- void consentWithCookie ( home , options . identity , baseUrl , index ) . catch ( ( ) => { } ) ;
240- }
241- } , 300 ) ;
242-
243- const exitCode = await new Promise < number | null > ( ( resolve ) => {
244- const killer = setTimeout ( ( ) => child . kill ( "SIGKILL" ) , options . timeoutMs ) ;
245- child . once ( "exit" , ( code ) => {
246- clearTimeout ( killer ) ;
247- resolve ( code ) ;
248- } ) ;
249- } ) ;
250- clearInterval ( consentLoop ) ;
251-
252- const events : TrialEvent [ ] = [ ] ;
253- for ( const line of stdout . split ( "\n" ) ) {
254- if ( ! line . trim ( ) ) continue ;
255- // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-json-parse -- boundary: tolerant parse of opencode's JSONL event stream
256- try {
257- events . push ( JSON . parse ( line ) as TrialEvent ) ;
258- } catch {
259- // Non-JSON line (banner, warning) — keep going.
260- }
261- }
262-
263- return {
264- events,
265- answerText : trialAnswerText ( events ) ,
266- rawStdout : stdout . length > 0 ? stdout : stderr ,
267- exitCode,
268- durationMs : Date . now ( ) - startedAt ,
269- } ;
270- } ) ;
89+ } ) ,
90+ ) ;
27191
27292// ---------------------------------------------------------------------------
27393// Task registry — a task is a prompt plus deterministic graders.
0 commit comments