Skip to content

Commit ec3d007

Browse files
HweinstockHweinstock
authored
test: add e2e test for evaluations lifecycle (#628)
1 parent afaec4f commit ec3d007

2 files changed

Lines changed: 249 additions & 47 deletions

File tree

‎e2e-tests/e2e-helper.ts‎

Lines changed: 52 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -20,14 +20,6 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
2020
const hasAws = hasAwsCredentials();
2121
const baseCanRun = prereqs.npm && prereqs.git && prereqs.uv && hasAws;
2222

23-
/**
24-
* Run the globally installed `agentcore` CLI.
25-
* E2E tests use the packaged binary to validate the published artifact.
26-
*/
27-
async function runAgentCore(args: string[], cwd: string): Promise<RunResult> {
28-
return spawnAndCollect('agentcore', args, cwd);
29-
}
30-
3123
interface E2EConfig {
3224
framework: string;
3325
modelProvider: string;
@@ -76,52 +68,19 @@ export function createE2ESuite(cfg: E2EConfig) {
7668
createArgs.push('--api-key', apiKey);
7769
}
7870

79-
const result = await runAgentCore(createArgs, testDir);
71+
const result = await runAgentCoreCLI(createArgs, testDir);
8072

8173
expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
8274
const json = parseJsonOutput(result.stdout) as { projectPath: string };
8375
projectPath = json.projectPath;
8476

85-
// TODO: Replace with `agentcore add target` once the CLI command is re-introduced
86-
const account =
87-
process.env.AWS_ACCOUNT_ID ??
88-
execSync('aws sts get-caller-identity --query Account --output text').toString().trim();
89-
const region = process.env.AWS_REGION ?? 'us-east-1';
90-
const awsTargetsPath = join(projectPath, 'agentcore', 'aws-targets.json');
91-
await writeFile(awsTargetsPath, JSON.stringify([{ name: 'default', account, region }]));
92-
93-
// Override @aws/agentcore-cdk with a local tarball if provided (for cross-package testing)
94-
if (process.env.CDK_TARBALL) {
95-
execSync(`npm install -f ${process.env.CDK_TARBALL}`, {
96-
cwd: join(projectPath, 'agentcore', 'cdk'),
97-
stdio: 'pipe',
98-
});
99-
}
77+
await writeAwsTargets(projectPath);
78+
installCdkTarball(projectPath);
10079
}, 300000);
10180

10281
afterAll(async () => {
10382
if (projectPath && hasAws) {
104-
await runAgentCore(['remove', 'all', '--json'], projectPath);
105-
const result = await runAgentCore(['deploy', '--yes', '--json'], projectPath);
106-
107-
if (result.exitCode !== 0) {
108-
console.log('Teardown stdout:', result.stdout);
109-
console.log('Teardown stderr:', result.stderr);
110-
}
111-
112-
// Delete the API key credential provider from the account.
113-
// These are created as a pre-deploy step outside CDK and are not
114-
// cleaned up by stack teardown, so we must delete them explicitly.
115-
if (cfg.modelProvider !== 'Bedrock' && agentName) {
116-
const providerName = `${agentName}${cfg.modelProvider}`;
117-
const region = process.env.AWS_REGION ?? 'us-east-1';
118-
try {
119-
const client = new BedrockAgentCoreControlClient({ region });
120-
await client.send(new DeleteApiKeyCredentialProviderCommand({ name: providerName }));
121-
} catch {
122-
// Best-effort cleanup — don't fail the test if deletion fails
123-
}
124-
}
83+
await teardownE2EProject(projectPath, agentName, cfg.modelProvider);
12584
}
12685
if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
12786
}, 600000);
@@ -138,7 +97,7 @@ export function createE2ESuite(cfg: E2EConfig) {
13897

13998
await retry(
14099
async () => {
141-
const result = await runAgentCore(['deploy', '--yes', '--json'], projectPath);
100+
const result = await runAgentCoreCLI(['deploy', '--yes', '--json'], projectPath);
142101

143102
if (result.exitCode !== 0) {
144103
console.log('Deploy stdout:', result.stdout);
@@ -165,7 +124,7 @@ export function createE2ESuite(cfg: E2EConfig) {
165124
// Retry invoke to handle cold-start / runtime initialization delays
166125
await retry(
167126
async () => {
168-
const result = await runAgentCore(
127+
const result = await runAgentCoreCLI(
169128
['invoke', '--prompt', 'Say hello', '--agent', agentName, '--json'],
170129
projectPath
171130
);
@@ -304,3 +263,49 @@ export function createE2ESuite(cfg: E2EConfig) {
304263
);
305264
});
306265
}
266+
267+
export { hasAws, baseCanRun };
268+
269+
export function runAgentCoreCLI(args: string[], cwd: string): Promise<RunResult> {
270+
return spawnAndCollect('agentcore', args, cwd);
271+
}
272+
273+
// TODO: Replace with `agentcore add target` once the CLI command is re-introduced
274+
export async function writeAwsTargets(projectPath: string): Promise<void> {
275+
const account =
276+
process.env.AWS_ACCOUNT_ID ??
277+
execSync('aws sts get-caller-identity --query Account --output text').toString().trim();
278+
const region = process.env.AWS_REGION ?? 'us-east-1';
279+
await writeFile(
280+
join(projectPath, 'agentcore', 'aws-targets.json'),
281+
JSON.stringify([{ name: 'default', account, region }])
282+
);
283+
}
284+
285+
export function installCdkTarball(projectPath: string): void {
286+
if (process.env.CDK_TARBALL) {
287+
execSync(`npm install -f ${process.env.CDK_TARBALL}`, {
288+
cwd: join(projectPath, 'agentcore', 'cdk'),
289+
stdio: 'pipe',
290+
});
291+
}
292+
}
293+
294+
export async function teardownE2EProject(projectPath: string, agentName: string, modelProvider: string): Promise<void> {
295+
await spawnAndCollect('agentcore', ['remove', 'all', '--json'], projectPath);
296+
const result = await spawnAndCollect('agentcore', ['deploy', '--yes', '--json'], projectPath);
297+
if (result.exitCode !== 0) {
298+
console.log('Teardown stdout:', result.stdout);
299+
console.log('Teardown stderr:', result.stderr);
300+
}
301+
if (modelProvider !== 'Bedrock' && agentName) {
302+
const providerName = `${agentName}${modelProvider}`;
303+
const region = process.env.AWS_REGION ?? 'us-east-1';
304+
try {
305+
const client = new BedrockAgentCoreControlClient({ region });
306+
await client.send(new DeleteApiKeyCredentialProviderCommand({ name: providerName }));
307+
} catch {
308+
// Best-effort cleanup
309+
}
310+
}
311+
}

‎e2e-tests/evals-lifecycle.test.ts‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
import { parseJsonOutput, retry } from '../src/test-utils/index.js';
2+
import {
3+
baseCanRun,
4+
hasAws,
5+
installCdkTarball,
6+
runAgentCoreCLI,
7+
teardownE2EProject,
8+
writeAwsTargets,
9+
} from './e2e-helper.js';
10+
import { randomUUID } from 'node:crypto';
11+
import { mkdir, rm } from 'node:fs/promises';
12+
import { tmpdir } from 'node:os';
13+
import { join } from 'node:path';
14+
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
15+
16+
const canRun = baseCanRun && hasAws;
17+
18+
describe.sequential('e2e: evaluations lifecycle', () => {
19+
let testDir: string;
20+
let projectPath: string;
21+
const agentName = `E2eEval${String(Date.now()).slice(-8)}`;
22+
const evalName = 'E2eEvaluator';
23+
const onlineEvalName = 'E2eOnlineEval';
24+
25+
beforeAll(async () => {
26+
if (!canRun) return;
27+
28+
testDir = join(tmpdir(), `agentcore-e2e-evals-${randomUUID()}`);
29+
await mkdir(testDir, { recursive: true });
30+
31+
const result = await runAgentCoreCLI(
32+
[
33+
'create',
34+
'--name',
35+
agentName,
36+
'--language',
37+
'Python',
38+
'--framework',
39+
'Strands',
40+
'--model-provider',
41+
'Bedrock',
42+
'--memory',
43+
'none',
44+
'--json',
45+
],
46+
testDir
47+
);
48+
expect(result.exitCode, `Create failed: ${result.stderr}`).toBe(0);
49+
projectPath = (parseJsonOutput(result.stdout) as { projectPath: string }).projectPath;
50+
51+
await writeAwsTargets(projectPath);
52+
installCdkTarball(projectPath);
53+
}, 300000);
54+
55+
afterAll(async () => {
56+
if (projectPath && hasAws) {
57+
await teardownE2EProject(projectPath, agentName, 'Bedrock');
58+
}
59+
if (testDir) await rm(testDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 1000 });
60+
}, 600000);
61+
62+
const run = (args: string[]) => runAgentCoreCLI(args, projectPath);
63+
64+
it.skipIf(!canRun)(
65+
'configures evaluator and online eval before deploy',
66+
async () => {
67+
let result = await run([
68+
'add',
69+
'evaluator',
70+
'--name',
71+
evalName,
72+
'--level',
73+
'SESSION',
74+
'--model',
75+
'us.anthropic.claude-sonnet-4-5-20250929-v1:0',
76+
'--instructions',
77+
'Evaluate the overall quality of this session. Context: {context}',
78+
'--json',
79+
]);
80+
expect(result.exitCode, `Add evaluator failed: ${result.stdout}`).toBe(0);
81+
82+
result = await run([
83+
'add',
84+
'online-eval',
85+
'--name',
86+
onlineEvalName,
87+
'--agent',
88+
agentName,
89+
'--evaluator',
90+
evalName,
91+
'--sampling-rate',
92+
'100',
93+
'--enable-on-create',
94+
'--json',
95+
]);
96+
expect(result.exitCode, `Add online-eval failed: ${result.stdout}`).toBe(0);
97+
},
98+
60000
99+
);
100+
101+
it.skipIf(!canRun)(
102+
'deploys agent with evaluator and online eval config',
103+
async () => {
104+
const result = await run(['deploy', '--yes', '--json']);
105+
if (result.exitCode !== 0) {
106+
console.log('Deploy stdout:', result.stdout);
107+
console.log('Deploy stderr:', result.stderr);
108+
}
109+
expect(result.exitCode, 'Deploy failed').toBe(0);
110+
const json = parseJsonOutput(result.stdout) as { success: boolean };
111+
expect(json.success).toBe(true);
112+
},
113+
600000
114+
);
115+
116+
it.skipIf(!canRun)(
117+
'invokes the deployed agent',
118+
async () => {
119+
await retry(
120+
async () => {
121+
const result = await run(['invoke', '--prompt', 'Say hello', '--agent', agentName, '--json']);
122+
expect(result.exitCode, `Invoke failed: ${result.stderr}`).toBe(0);
123+
const json = parseJsonOutput(result.stdout) as { success: boolean };
124+
expect(json.success).toBe(true);
125+
},
126+
3,
127+
15000
128+
);
129+
},
130+
180000
131+
);
132+
133+
it.skipIf(!canRun)(
134+
'runs on-demand evaluation against agent traces',
135+
async () => {
136+
await retry(
137+
async () => {
138+
const result = await run([
139+
'run',
140+
'evals',
141+
'--agent',
142+
agentName,
143+
'--evaluator',
144+
'Builtin.Faithfulness',
145+
'--days',
146+
'1',
147+
'--json',
148+
]);
149+
expect(result.exitCode, `Run evals failed (stdout: ${result.stdout}, stderr: ${result.stderr})`).toBe(0);
150+
const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
151+
expect(json).toHaveProperty('success', true);
152+
expect(json).toHaveProperty('run');
153+
expect(json).toHaveProperty('filePath');
154+
},
155+
18,
156+
10000
157+
);
158+
},
159+
300000
160+
);
161+
162+
it.skipIf(!canRun)(
163+
'eval history shows the completed run',
164+
async () => {
165+
const result = await run(['evals', 'history', '--agent', agentName, '--json']);
166+
expect(result.exitCode, `Evals history failed: ${result.stderr}`).toBe(0);
167+
const json = parseJsonOutput(result.stdout) as Record<string, unknown> & { runs: unknown[] };
168+
expect(json).toHaveProperty('success', true);
169+
expect(json.runs.length, 'Should have at least one eval run').toBeGreaterThan(0);
170+
},
171+
120000
172+
);
173+
174+
it.skipIf(!canRun)(
175+
'pauses the online eval config',
176+
async () => {
177+
const result = await run(['pause', 'online-eval', onlineEvalName, '--json']);
178+
expect(result.exitCode, `Pause failed: ${result.stderr}`).toBe(0);
179+
const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
180+
expect(json).toHaveProperty('success', true);
181+
expect(json).toHaveProperty('executionStatus', 'DISABLED');
182+
},
183+
120000
184+
);
185+
186+
it.skipIf(!canRun)(
187+
'resumes the online eval config',
188+
async () => {
189+
const result = await run(['resume', 'online-eval', onlineEvalName, '--json']);
190+
expect(result.exitCode, `Resume failed: ${result.stderr}`).toBe(0);
191+
const json = parseJsonOutput(result.stdout) as Record<string, unknown>;
192+
expect(json).toHaveProperty('success', true);
193+
expect(json).toHaveProperty('executionStatus', 'ENABLED');
194+
},
195+
120000
196+
);
197+
});

0 commit comments

Comments
 (0)