Skip to content

Commit 64579d8

Browse files
authored
feat(runtime): execute docker environment recipes (#1676)
Bead: av-noh3.2.4
1 parent f50e507 commit 64579d8

6 files changed

Lines changed: 652 additions & 44 deletions

File tree

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
import { createHash } from 'node:crypto';
2+
import { tmpdir } from 'node:os';
3+
import path from 'node:path';
4+
5+
import type {
6+
DockerEnvironmentRecipe,
7+
EnvironmentSetupConfig,
8+
} from '../loaders/environment-recipe.js';
9+
import type { TargetRuntimeConfig } from '../providers/sandbox-runner.js';
10+
import type { JsonObject } from '../types.js';
11+
import {
12+
type CommandExecutor,
13+
DefaultCommandExecutor,
14+
DockerWorkspaceProvider,
15+
} from '../workspace/docker-workspace.js';
16+
17+
export type DockerEnvironmentSetupStatus = 'skipped' | 'success' | 'failed';
18+
19+
export interface DockerEnvironmentSetupResult {
20+
readonly type: 'docker';
21+
readonly image: string;
22+
readonly workdir: string;
23+
readonly status: DockerEnvironmentSetupStatus;
24+
readonly command?: readonly string[] | string;
25+
readonly cwd?: string;
26+
readonly args?: JsonObject;
27+
readonly stdout?: string;
28+
readonly stderr?: string;
29+
readonly exitCode?: number;
30+
readonly targetRuntime: TargetRuntimeConfig;
31+
}
32+
33+
export class DockerEnvironmentSetupError extends Error {
34+
readonly result: DockerEnvironmentSetupResult;
35+
36+
constructor(message: string, result: DockerEnvironmentSetupResult) {
37+
super(message);
38+
this.name = 'DockerEnvironmentSetupError';
39+
this.result = result;
40+
}
41+
}
42+
43+
function dockerImageTag(recipe: DockerEnvironmentRecipe): string {
44+
const hash = createHash('sha256')
45+
.update(recipe.context ?? '')
46+
.update('\0')
47+
.update(recipe.dockerfile ?? '')
48+
.digest('hex')
49+
.slice(0, 16);
50+
return `agentv-environment:${hash}`;
51+
}
52+
53+
function shellQuote(value: string): string {
54+
return `'${value.replaceAll("'", "'\\''")}'`;
55+
}
56+
57+
function setupPayload(recipe: DockerEnvironmentRecipe): JsonObject {
58+
return {
59+
args: recipe.setup?.args ?? {},
60+
environment: {
61+
type: 'docker',
62+
workdir: recipe.workdir,
63+
...(recipe.image ? { image: recipe.image } : {}),
64+
...(recipe.context ? { context: recipe.context } : {}),
65+
...(recipe.dockerfile ? { dockerfile: recipe.dockerfile } : {}),
66+
},
67+
...(recipe.setup?.env ? { env: recipe.setup.env } : {}),
68+
};
69+
}
70+
71+
function setupCommandLine(setup: EnvironmentSetupConfig, payload: string): string {
72+
const envPrefix = Object.entries(setup.env ?? {})
73+
.map(([key, value]) => `${key}=${shellQuote(value)}`)
74+
.join(' ');
75+
const command =
76+
typeof setup.command === 'string'
77+
? setup.command
78+
: setup.command.map((entry) => shellQuote(entry)).join(' ');
79+
const setupCommand = envPrefix ? `${envPrefix} ${command}` : command;
80+
return `printf %s ${shellQuote(payload)} | (${setupCommand})`;
81+
}
82+
83+
function setupTimeoutSeconds(setup: EnvironmentSetupConfig | undefined): number | undefined {
84+
return setup?.timeout_seconds;
85+
}
86+
87+
function dockerRuntimeForRecipe(
88+
recipe: DockerEnvironmentRecipe,
89+
image: string,
90+
): TargetRuntimeConfig {
91+
const setup = recipe.setup;
92+
const payload = JSON.stringify(setupPayload({ ...recipe, image }), null, 2);
93+
const tempDir = path.resolve(tmpdir());
94+
const recipeMounts = (recipe.mounts ?? []).map((mount) => ({
95+
source: mount.source,
96+
target: mount.target,
97+
access: mount.read_only === true ? 'ro' : (mount.access ?? 'rw'),
98+
}));
99+
const mounts = recipeMounts.some((mount) => mount.target === tempDir)
100+
? recipeMounts
101+
: [...recipeMounts, { source: tempDir, target: tempDir, access: 'rw' }];
102+
return {
103+
mode: 'sandbox',
104+
engine: 'docker',
105+
image,
106+
workdir: recipe.workdir,
107+
host_cwd: recipe.sourceDir,
108+
env: {
109+
...(recipe.env ?? {}),
110+
AGENTV_ENVIRONMENT_WORKDIR: recipe.workdir,
111+
},
112+
...(recipe.secrets !== undefined ? { secrets: recipe.secrets } : {}),
113+
mounts,
114+
...(recipe.resources?.memory !== undefined ? { memory: recipe.resources.memory } : {}),
115+
...(recipe.resources?.cpus !== undefined ? { cpus: recipe.resources.cpus } : {}),
116+
...(setup !== undefined ? { setup: [setupCommandLine(setup, payload)] } : {}),
117+
...(setupTimeoutSeconds(setup) !== undefined
118+
? { setup_timeout: setupTimeoutSeconds(setup) }
119+
: {}),
120+
};
121+
}
122+
123+
function formatDockerFailure(
124+
action: string,
125+
result: { stderr: string; stdout: string; exitCode: number },
126+
) {
127+
const details = result.stderr.trim() || result.stdout.trim();
128+
return details
129+
? `docker ${action} failed (exit ${result.exitCode}): ${details}`
130+
: `docker ${action} failed (exit ${result.exitCode})`;
131+
}
132+
133+
export async function prepareDockerEnvironment(
134+
recipe: DockerEnvironmentRecipe,
135+
executor: CommandExecutor = new DefaultCommandExecutor(),
136+
): Promise<DockerEnvironmentSetupResult> {
137+
const image = recipe.context ? (recipe.image ?? dockerImageTag(recipe)) : recipe.image;
138+
if (!image) {
139+
throw new Error('Docker environment requires image when context is not set.');
140+
}
141+
142+
const docker = new DockerWorkspaceProvider(
143+
{
144+
image,
145+
...(recipe.resources?.memory !== undefined ? { memory: recipe.resources.memory } : {}),
146+
...(recipe.resources?.cpus !== undefined ? { cpus: recipe.resources.cpus } : {}),
147+
},
148+
executor,
149+
);
150+
151+
if (!(await docker.isDockerAvailable())) {
152+
throw new Error(
153+
'Docker environment configured but Docker CLI is not available. Install Docker and ensure it is running.',
154+
);
155+
}
156+
157+
if (recipe.context) {
158+
const argv = ['docker', 'build', '-t', image];
159+
if (recipe.dockerfile) {
160+
argv.push('-f', recipe.dockerfile);
161+
}
162+
argv.push(recipe.context);
163+
const result = await executor.exec(argv, { timeoutMs: 30 * 60 * 1000 });
164+
if (result.exitCode !== 0) {
165+
const failure: DockerEnvironmentSetupResult = {
166+
type: 'docker',
167+
image,
168+
workdir: recipe.workdir,
169+
status: 'failed',
170+
stderr: result.stderr,
171+
stdout: result.stdout,
172+
exitCode: result.exitCode,
173+
targetRuntime: dockerRuntimeForRecipe(recipe, image),
174+
};
175+
throw new DockerEnvironmentSetupError(formatDockerFailure('build', result), failure);
176+
}
177+
} else {
178+
await docker.pullImage();
179+
}
180+
181+
const runtime = dockerRuntimeForRecipe(recipe, image);
182+
return {
183+
type: 'docker',
184+
image,
185+
workdir: recipe.workdir,
186+
status: recipe.setup ? 'success' : 'skipped',
187+
...(recipe.setup?.command !== undefined ? { command: recipe.setup.command } : {}),
188+
cwd: recipe.sourceDir,
189+
...(recipe.setup?.args !== undefined ? { args: recipe.setup.args } : {}),
190+
targetRuntime: runtime,
191+
};
192+
}

packages/core/src/evaluation/orchestrator.ts

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ export interface EvaluationCache {
419419
export interface RunEvalCaseOptions {
420420
readonly evalCase: EvalTest;
421421
readonly provider: Provider;
422+
readonly providerFactory?: (target: ResolvedTarget) => Provider;
422423
readonly target: ResolvedTarget;
423424
readonly evaluators: Partial<Record<string, Grader>> & { readonly 'llm-grader': Grader };
424425
readonly now?: () => Date;
@@ -1257,6 +1258,7 @@ export async function runEvaluation(
12571258
const runCaseOptions: RunEvalCaseOptions = {
12581259
evalCase: evalCase,
12591260
provider: primaryProvider,
1261+
providerFactory,
12601262
target,
12611263
evaluators: evaluatorRegistry,
12621264
maxRetries,
@@ -1874,12 +1876,6 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
18741876
const promptInputs = await buildPromptInputs(evalCase, formattingMode);
18751877
const typeRegistry = providedTypeRegistry ?? createBuiltinRegistry();
18761878

1877-
const cacheKey = useCache ? createCacheKey(provider, target, evalCase, promptInputs) : undefined;
1878-
let cachedResponse: ProviderResponse | undefined;
1879-
if (cacheKey && cache) {
1880-
cachedResponse = await cache.get(cacheKey);
1881-
}
1882-
18831879
const nowFn = now ?? (() => new Date());
18841880

18851881
let afterEachOutput: string | undefined;
@@ -1921,10 +1917,24 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
19211917
beforeAllOutput,
19221918
beforeEachOutput,
19231919
baselineCommit,
1920+
targetCwd,
1921+
targetRuntime,
19241922
isSharedWorkspace,
19251923
caseWorkspaceFile,
19261924
extensionState,
19271925
} = workspaceSetup;
1926+
const effectiveTarget = targetRuntime ? { ...target, runtime: targetRuntime } : target;
1927+
const effectiveProvider =
1928+
targetRuntime !== undefined
1929+
? (options.providerFactory ?? createProvider)(effectiveTarget)
1930+
: provider;
1931+
const cacheKey = useCache
1932+
? createCacheKey(effectiveProvider, effectiveTarget, evalCase, promptInputs)
1933+
: undefined;
1934+
let cachedResponse: ProviderResponse | undefined;
1935+
if (cacheKey && cache) {
1936+
cachedResponse = await cache.get(cacheKey);
1937+
}
19281938
const extensionMetadata = extensionState?.metadata;
19291939
const providerMetadata = mergeMetadata(evalCase.metadata, extensionState?.providerContext);
19301940
const resultMetadata = mergeMetadata(evalCase.metadata, extensionMetadata);
@@ -2032,15 +2042,15 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
20322042
if (evalCase.mode === 'conversation' && evalCase.turns?.length) {
20332043
let conversationResult = await runConversationMode({
20342044
evalCase,
2035-
provider,
2036-
target,
2045+
provider: effectiveProvider,
2046+
target: effectiveTarget,
20372047
evaluators,
20382048
typeRegistry,
20392049
graderProvider,
20402050
promptInputs,
20412051
nowFn,
20422052
signal,
2043-
workspacePath,
2053+
workspacePath: targetCwd ?? workspacePath,
20442054
caseWorkspaceFile,
20452055
agentTimeoutMs,
20462056
streamCallbacks: options.streamCallbacks,
@@ -2085,15 +2095,15 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
20852095

20862096
while (!providerResponse && attempt < attemptBudget) {
20872097
try {
2088-
providerResponse = await invokeProvider(provider, {
2098+
providerResponse = await invokeProvider(effectiveProvider, {
20892099
evalCase: evalCase,
2090-
target,
2100+
target: effectiveTarget,
20912101
promptInputs,
20922102
attempt,
20932103
evalFilePath,
20942104
agentTimeoutMs,
20952105
signal,
2096-
cwd: workspacePath,
2106+
cwd: targetCwd ?? workspacePath,
20972107
workspaceFile: caseWorkspaceFile,
20982108
captureFileChanges: !!baselineCommit,
20992109
streamCallbacks: options.streamCallbacks,
@@ -2112,22 +2122,22 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
21122122
}
21132123

21142124
// Try fallback targets in order after exhausting retries on the primary
2115-
if (!providerResponse && target.fallbackTargets?.length && targetResolver) {
2116-
for (const fallbackName of target.fallbackTargets) {
2125+
if (!providerResponse && effectiveTarget.fallbackTargets?.length && targetResolver) {
2126+
for (const fallbackName of effectiveTarget.fallbackTargets) {
21172127
const fallbackProvider = targetResolver(fallbackName);
21182128
if (!fallbackProvider) {
21192129
continue;
21202130
}
21212131
try {
21222132
providerResponse = await invokeProvider(fallbackProvider, {
21232133
evalCase: evalCase,
2124-
target,
2134+
target: effectiveTarget,
21252135
promptInputs,
21262136
attempt: 0,
21272137
evalFilePath,
21282138
agentTimeoutMs,
21292139
signal,
2130-
cwd: workspacePath,
2140+
cwd: targetCwd ?? workspacePath,
21312141
workspaceFile: caseWorkspaceFile,
21322142
captureFileChanges: !!baselineCommit,
21332143
streamCallbacks: options.streamCallbacks,
@@ -2145,11 +2155,11 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
21452155
if (!providerResponse) {
21462156
const errorResult = buildErrorResult(
21472157
evalCase,
2148-
target.name,
2158+
effectiveTarget.name,
21492159
nowFn(),
21502160
lastError ?? new Error('Provider did not return a response'),
21512161
promptInputs,
2152-
provider,
2162+
effectiveProvider,
21532163
'agent',
21542164
'provider_error',
21552165
verbose,
@@ -2172,7 +2182,7 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
21722182
evalCase,
21732183
evalFilePath,
21742184
repoRoot,
2175-
target: targetUsed ? { ...target, name: targetUsed } : target,
2185+
target: targetUsed ? { ...effectiveTarget, name: targetUsed } : effectiveTarget,
21762186
attempt,
21772187
response: providerResponse,
21782188
nowFn,
@@ -2247,8 +2257,8 @@ export async function runEvalCase(options: RunEvalCaseOptions): Promise<Evaluati
22472257
const result = await evaluateCandidate({
22482258
evalCase,
22492259
candidate,
2250-
target,
2251-
provider,
2260+
target: effectiveTarget,
2261+
provider: effectiveProvider,
22522262
evaluators,
22532263
typeRegistry,
22542264
promptInputs,

packages/core/src/evaluation/providers/sandbox-runner.ts

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,10 @@ function dockerWorkdir(runtime: TargetRuntimeConfig): string | undefined {
107107
return asString(runtime.workdir) ?? asString(runtime.workspace);
108108
}
109109

110+
function dockerHostCwd(runtime: TargetRuntimeConfig): string | undefined {
111+
return asString(runtime.host_cwd);
112+
}
113+
110114
function dockerNetwork(runtime: TargetRuntimeConfig): string {
111115
const network = asString(runtime.network);
112116
return network ?? 'none';
@@ -116,6 +120,14 @@ function dockerSetupCommands(runtime: TargetRuntimeConfig): readonly string[] {
116120
return asStringArray(runtime.setup);
117121
}
118122

123+
function dockerMemory(runtime: TargetRuntimeConfig): string | undefined {
124+
return asString(runtime.memory);
125+
}
126+
127+
function dockerCpus(runtime: TargetRuntimeConfig): number | undefined {
128+
return typeof runtime.cpus === 'number' && runtime.cpus > 0 ? runtime.cpus : undefined;
129+
}
130+
119131
export async function runDockerSandboxCommand(
120132
command: string,
121133
options: SandboxCommandRunOptions,
@@ -159,6 +171,16 @@ export async function runDockerSandboxCommand(
159171
argv.push('--workdir', workdir);
160172
}
161173

174+
const memory = dockerMemory(options.runtime);
175+
if (memory) {
176+
argv.push('--memory', memory);
177+
}
178+
179+
const cpus = dockerCpus(options.runtime);
180+
if (cpus !== undefined) {
181+
argv.push('--cpus', String(cpus));
182+
}
183+
162184
for (const [key, value] of Object.entries(dockerEnv(options.runtime))) {
163185
argv.push('--env', `${key}=${value}`);
164186
}
@@ -172,7 +194,7 @@ export async function runDockerSandboxCommand(
172194

173195
return new Promise((resolve) => {
174196
const child = spawn('docker', argv, {
175-
cwd: options.cwd,
197+
cwd: dockerHostCwd(options.runtime) ?? options.cwd,
176198
env: process.env,
177199
windowsHide: true,
178200
});

0 commit comments

Comments
 (0)