Skip to content

Commit b4fe341

Browse files
committed
fix: restore preview's integration tests for create/evaluator
Main's integration tests expect telemetry instrumentation in the create CLI path which preview doesn't have (preview's create/command.tsx has the harness path instead of main's telemetry wrapping). Use preview's test versions which match preview's behavior.
1 parent 4bc3fde commit b4fe341

3 files changed

Lines changed: 5 additions & 45 deletions

File tree

integ-tests/add-remove-evaluator.test.ts

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,10 @@
11
import { createTestProject, parseJsonOutput, readProjectConfig, runCLI } from '../src/test-utils/index.js';
22
import type { TestProject } from '../src/test-utils/index.js';
3-
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
43
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
54

6-
const telemetry = createTelemetryHelper();
7-
85
/** Run a CLI command and assert it succeeds, returning parsed JSON output. */
96
async function runSuccess(args: string[], cwd: string) {
10-
const result = await runCLI(args, cwd, { env: telemetry.env });
7+
const result = await runCLI(args, cwd);
118
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
129
const json: unknown = parseJsonOutput(result.stdout);
1310
expect(json).toHaveProperty('success', true);
@@ -16,7 +13,7 @@ async function runSuccess(args: string[], cwd: string) {
1613

1714
/** Run a CLI command and assert it fails, returning parsed JSON output. */
1815
async function runFailure(args: string[], cwd: string) {
19-
const result = await runCLI(args, cwd, { env: telemetry.env });
16+
const result = await runCLI(args, cwd);
2017
expect(result.exitCode).toBe(1);
2118
const json: unknown = parseJsonOutput(result.stdout);
2219
expect(json).toHaveProperty('success', false);
@@ -38,7 +35,6 @@ describe('integration: add and remove evaluators and online eval configs', () =>
3835

3936
afterAll(async () => {
4037
await project.cleanup();
41-
telemetry.destroy();
4238
});
4339

4440
describe('evaluator and online eval lifecycle', () => {
@@ -127,29 +123,25 @@ describe('integration: add and remove evaluators and online eval configs', () =>
127123

128124
const config = await readProjectConfig(project.projectPath);
129125
expect(config.onlineEvalConfigs.find(c => c.name === configName)).toBeUndefined();
130-
telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'success' });
131126
});
132127

133128
it('removes the evaluator after online eval is gone', async () => {
134129
await runSuccess(['remove', 'evaluator', '--name', evalName, '--json'], project.projectPath);
135130

136131
const config = await readProjectConfig(project.projectPath);
137132
expect(config.evaluators.find(e => e.name === evalName)).toBeUndefined();
138-
telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'success' });
139133
});
140134
});
141135

142136
describe('error cases', () => {
143137
it('fails to remove non-existent evaluator', async () => {
144138
const json = await runFailure(['remove', 'evaluator', '--name', 'NonExistent', '--json'], project.projectPath);
145139
expect(json.error).toContain('not found');
146-
telemetry.assertMetricEmitted({ command: 'remove.evaluator', exit_reason: 'failure' });
147140
});
148141

149142
it('fails to remove non-existent online eval config', async () => {
150143
const json = await runFailure(['remove', 'online-eval', '--name', 'NonExistent', '--json'], project.projectPath);
151144
expect(json.error).toContain('not found');
152-
telemetry.assertMetricEmitted({ command: 'remove.online-eval', exit_reason: 'failure' });
153145
});
154146

155147
it('rejects evaluator with missing --level', async () => {

integ-tests/create-edge-cases.test.ts

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
/* eslint-disable security/detect-non-literal-fs-filename */
22
import { exists, prereqs, runCLI } from '../src/test-utils/index.js';
3-
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
43
import { randomUUID } from 'node:crypto';
54
import { mkdir, rm } from 'node:fs/promises';
65
import { tmpdir } from 'node:os';
@@ -10,21 +9,18 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
109
describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', () => {
1110
let testDir: string;
1211

13-
const telemetry = createTelemetryHelper();
14-
1512
beforeAll(async () => {
1613
testDir = join(tmpdir(), `agentcore-integ-edge-${randomUUID()}`);
1714
await mkdir(testDir, { recursive: true });
1815
});
1916

2017
afterAll(async () => {
21-
telemetry.destroy();
2218
await rm(testDir, { recursive: true, force: true });
2319
});
2420

2521
describe('reserved names', () => {
2622
it('rejects reserved name "Test"', async () => {
27-
const result = await runCLI(['create', '--name', 'Test', '--json'], testDir, { env: telemetry.env });
23+
const result = await runCLI(['create', '--name', 'Test', '--json'], testDir);
2824

2925
expect(result.exitCode).toBe(1);
3026
const json = JSON.parse(result.stdout);
@@ -34,11 +30,6 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
3430
json.error.toLowerCase().includes('reserved') || json.error.toLowerCase().includes('conflict'),
3531
`Error should mention reserved/conflict: ${json.error}`
3632
).toBeTruthy();
37-
38-
telemetry.assertMetricEmitted({
39-
command: 'create',
40-
exit_reason: 'failure',
41-
});
4233
});
4334

4435
it('rejects reserved name "bedrock"', async () => {
@@ -130,21 +121,12 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
130121
describe('flag interactions', () => {
131122
it('--defaults creates project with default settings', async () => {
132123
const name = `Def${Date.now().toString().slice(-6)}`;
133-
const result = await runCLI(['create', '--name', name, '--defaults', '--json'], testDir, { env: telemetry.env });
124+
const result = await runCLI(['create', '--name', name, '--defaults', '--json'], testDir);
134125

135126
expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
136127
const json = JSON.parse(result.stdout);
137128
expect(json.success).toBe(true);
138129
expect(json.projectPath).toBeTruthy();
139-
140-
telemetry.assertMetricEmitted({
141-
command: 'create',
142-
exit_reason: 'success',
143-
language: 'python',
144-
framework: 'strands',
145-
model_provider: 'bedrock',
146-
has_agent: 'true',
147-
});
148130
});
149131

150132
it('--dry-run shows what would be created without writing files', async () => {

integ-tests/create-frameworks.test.ts

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
import { exists, prereqs, readProjectConfig, runCLI } from '../src/test-utils/index.js';
2-
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
32
import { randomUUID } from 'node:crypto';
43
import { mkdir, readFile, rm } from 'node:fs/promises';
54
import { tmpdir } from 'node:os';
@@ -9,15 +8,12 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
98
describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with different frameworks', () => {
109
let testDir: string;
1110

12-
const telemetry = createTelemetryHelper();
13-
1411
beforeAll(async () => {
1512
testDir = join(tmpdir(), `agentcore-integ-frameworks-${randomUUID()}`);
1613
await mkdir(testDir, { recursive: true });
1714
});
1815

1916
afterAll(async () => {
20-
telemetry.destroy();
2117
await rm(testDir, { recursive: true, force: true });
2218
});
2319

@@ -38,8 +34,7 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
3834
'none',
3935
'--json',
4036
],
41-
testDir,
42-
{ env: telemetry.env }
37+
testDir
4338
);
4439

4540
expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
@@ -65,15 +60,6 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
6560
expect(agents).toBeDefined();
6661
expect(agents.length).toBe(1);
6762
expect(agents[0]!.name).toBe(agentName);
68-
69-
telemetry.assertMetricEmitted({
70-
command: 'create',
71-
exit_reason: 'success',
72-
language: 'python',
73-
framework: 'langchain_langgraph',
74-
model_provider: 'bedrock',
75-
has_agent: 'true',
76-
});
7763
});
7864

7965
it('creates GoogleADK project with Gemini provider', async () => {

0 commit comments

Comments
 (0)