|
1 | 1 | import { Output, generateText } from 'ai' |
2 | | -import { createOpencode } from 'ai-sdk-provider-opencode-sdk' |
3 | 2 | import { z } from 'zod' |
4 | 3 | import { |
5 | 4 | collectOpencodeSessionSnapshot, |
6 | 5 | type OpencodeSessionSnapshot, |
7 | 6 | } from 'runner/utils/opencode-session' |
8 | | -import { ensureOpencodeServerStarted } from 'runner/utils/opencode' |
9 | | - |
| 7 | +import { createIsolatedOpencodeModel } from 'runner/utils/opencode-model' |
| 8 | +import { |
| 9 | + logOpencodeTrace, |
| 10 | + runTracedOpencodeCall, |
| 11 | + startOpencodeCallTracer, |
| 12 | +} from 'runner/utils/opencode-trace' |
10 | 13 | const JSON_FALLBACK_SYSTEM_PROMPT = ` |
11 | 14 | Return only valid JSON matching this shape: |
12 | 15 | { |
@@ -51,6 +54,8 @@ type RunJudgeCallOptions = { |
51 | 54 | timeout: number |
52 | 55 | port?: number |
53 | 56 | directory?: string |
| 57 | + cwd?: string |
| 58 | + verbose?: boolean |
54 | 59 | } |
55 | 60 |
|
56 | 61 | function asRecord(value: unknown) { |
@@ -110,63 +115,101 @@ function parseJudgeOutputFromText(rawText: string) { |
110 | 115 | export async function runJudgeCall( |
111 | 116 | options: RunJudgeCallOptions |
112 | 117 | ): Promise<JudgeCallResult> { |
113 | | - await ensureOpencodeServerStarted({ timeout: options.timeout, port: options.port }) |
| 118 | + if (options.port === undefined) { |
| 119 | + throw new Error('runJudgeCall requires an opencode server port') |
| 120 | + } |
114 | 121 |
|
115 | | - const provider = createOpencode({ |
116 | | - autoStartServer: false, |
| 122 | + const promptBytes = Buffer.byteLength(options.prompt, 'utf8') |
| 123 | + const tracer = startOpencodeCallTracer({ |
| 124 | + label: 'judge', |
| 125 | + model: options.model, |
117 | 126 | port: options.port, |
| 127 | + directory: options.directory ?? options.cwd, |
| 128 | + timeoutMs: options.timeout, |
| 129 | + inputSummary: `promptBytes=${promptBytes}`, |
118 | 130 | }) |
119 | 131 |
|
120 | | - const judgeModel = provider(options.model, { createNewSession: true }) |
| 132 | + const { model: judgeModel, dispose } = createIsolatedOpencodeModel( |
| 133 | + options.model, |
| 134 | + { |
| 135 | + port: options.port, |
| 136 | + cwd: options.cwd, |
| 137 | + } |
| 138 | + ) |
121 | 139 |
|
122 | 140 | try { |
123 | | - const response = await generateText({ |
124 | | - model: judgeModel, |
125 | | - prompt: options.prompt, |
126 | | - abortSignal: AbortSignal.timeout(options.timeout), |
127 | | - output: Output.object({ |
128 | | - schema: structuredOutputSchema, |
129 | | - name: 'eval_requirements_result', |
130 | | - description: 'Requirement verdicts for a React Native eval', |
131 | | - }), |
132 | | - }) |
| 141 | + try { |
| 142 | + const response = await runTracedOpencodeCall( |
| 143 | + tracer, |
| 144 | + 'generateText:structured-output', |
| 145 | + () => |
| 146 | + generateText({ |
| 147 | + model: judgeModel, |
| 148 | + prompt: options.prompt, |
| 149 | + abortSignal: AbortSignal.timeout(options.timeout), |
| 150 | + output: Output.object({ |
| 151 | + schema: structuredOutputSchema, |
| 152 | + name: 'eval_requirements_result', |
| 153 | + description: 'Requirement verdicts for a React Native eval', |
| 154 | + }), |
| 155 | + }) |
| 156 | + ) |
133 | 157 |
|
134 | | - return { |
135 | | - summary: response.output.summary, |
136 | | - requirements: response.output.requirements, |
137 | | - opencodeSession: await collectOpencodeSessionSnapshot({ |
138 | | - sessionId: extractOpencodeSessionId(response), |
139 | | - port: options.port, |
140 | | - directory: options.directory, |
141 | | - }), |
142 | | - } |
143 | | - } catch (structuredOutputError) { |
144 | | - const fallbackResponse = await generateText({ |
145 | | - model: judgeModel, |
146 | | - prompt: options.prompt, |
147 | | - system: JSON_FALLBACK_SYSTEM_PROMPT, |
148 | | - abortSignal: AbortSignal.timeout(options.timeout), |
149 | | - }) |
| 158 | + tracer.noteSessionId(extractOpencodeSessionId(response)) |
150 | 159 |
|
151 | | - const parsedOutput = parseJudgeOutputFromText(fallbackResponse.text) |
152 | | - if (!parsedOutput.success) { |
153 | | - const originalMessage = |
154 | | - structuredOutputError instanceof Error |
155 | | - ? structuredOutputError.message |
156 | | - : String(structuredOutputError) |
157 | | - throw new Error( |
158 | | - `judge did not return valid requirement output (structured output failed: ${originalMessage})` |
| 160 | + return { |
| 161 | + summary: response.output.summary, |
| 162 | + requirements: response.output.requirements, |
| 163 | + opencodeSession: await collectOpencodeSessionSnapshot({ |
| 164 | + sessionId: extractOpencodeSessionId(response), |
| 165 | + port: options.port, |
| 166 | + directory: options.directory, |
| 167 | + }), |
| 168 | + } |
| 169 | + } catch (structuredOutputError) { |
| 170 | + logOpencodeTrace( |
| 171 | + `structured output failed; trying JSON fallback: ${structuredOutputError instanceof Error ? structuredOutputError.message : String(structuredOutputError)}` |
159 | 172 | ) |
160 | | - } |
161 | 173 |
|
162 | | - return { |
163 | | - summary: parsedOutput.data.summary, |
164 | | - requirements: parsedOutput.data.requirements, |
165 | | - opencodeSession: await collectOpencodeSessionSnapshot({ |
166 | | - sessionId: extractOpencodeSessionId(fallbackResponse), |
167 | | - port: options.port, |
168 | | - directory: options.directory, |
169 | | - }), |
| 174 | + const fallbackResponse = await runTracedOpencodeCall( |
| 175 | + tracer, |
| 176 | + 'generateText:json-fallback', |
| 177 | + () => |
| 178 | + generateText({ |
| 179 | + model: judgeModel, |
| 180 | + prompt: options.prompt, |
| 181 | + system: JSON_FALLBACK_SYSTEM_PROMPT, |
| 182 | + abortSignal: AbortSignal.timeout(options.timeout), |
| 183 | + }) |
| 184 | + ) |
| 185 | + |
| 186 | + tracer.noteSessionId( |
| 187 | + extractOpencodeSessionId(fallbackResponse) |
| 188 | + ) |
| 189 | + |
| 190 | + const parsedOutput = parseJudgeOutputFromText(fallbackResponse.text) |
| 191 | + if (!parsedOutput.success) { |
| 192 | + const originalMessage = |
| 193 | + structuredOutputError instanceof Error |
| 194 | + ? structuredOutputError.message |
| 195 | + : String(structuredOutputError) |
| 196 | + throw new Error( |
| 197 | + `judge did not return valid requirement output (structured output failed: ${originalMessage})` |
| 198 | + ) |
| 199 | + } |
| 200 | + |
| 201 | + return { |
| 202 | + summary: parsedOutput.data.summary, |
| 203 | + requirements: parsedOutput.data.requirements, |
| 204 | + opencodeSession: await collectOpencodeSessionSnapshot({ |
| 205 | + sessionId: extractOpencodeSessionId(fallbackResponse), |
| 206 | + port: options.port, |
| 207 | + directory: options.directory, |
| 208 | + }), |
| 209 | + } |
170 | 210 | } |
| 211 | + } finally { |
| 212 | + tracer.stop() |
| 213 | + await dispose() |
171 | 214 | } |
172 | 215 | } |
0 commit comments