Skip to content

Commit e9dfc16

Browse files
HweinstockHweinstock
authored
feat: wire telemetry into all add.* commands (#1050)
* feat: wire telemetry withCommandRun into all add.* commands * refactor: extract cliCommandRun helper, apply to all add.* primitives * test: add audit file assertions for all add.* telemetry * test: add telemetry audit assertions to existing add integ tests * refactor: extract shared audit test utils into src/test-utils/audit.ts * fix: address review feedback — guard telemetry init, replaceAll, unknown fallback, TUI try/catch * fix: AgentPrimitive TUI try/catch, standardize uses safeParse * refactor: extract standalone assertTelemetry helper * refactor: rename audit.ts to telemetry-helper.ts, clarify method names * refactor: move assertTelemetry into TelemetryHelper as assertMetricEmitted * feat: add telemetry to TUI add paths via withAddTelemetry * fix: review feedback — withAddTelemetry safety, standardize handles undefined, MCP agent attrs, policy TUI attrs * fix: remove unnecessary type assertion * fix: address review — document standardize cast, add policy-engine + episodic telemetry tests * refactor: centralize gateway target type mapping in common-shapes * fix: preserve original function error with telemetry wrapper * refactor: extract telemetryAttrs into a single line * feat: wire up telemetry for addAgent
1 parent 3dccd97 commit e9dfc16

31 files changed

Lines changed: 817 additions & 447 deletions

‎e2e-tests/byo-custom-jwt.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ const region = process.env.AWS_REGION ?? 'us-east-1';
4848
* Run the local CLI build without skipping install (needed for deploy).
4949
*/
5050
function runLocalCLI(args: string[], cwd: string): Promise<RunResult> {
51-
return runCLI(args, cwd, /* skipInstall */ false);
51+
return runCLI(args, cwd, { skipInstall: false });
5252
}
5353

5454
describe.sequential('e2e: BYO agent with CUSTOM_JWT auth', () => {

‎integ-tests/add-remove-resources.test.ts‎

Lines changed: 55 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import { createTestProject, readProjectConfig, runCLI } from '../src/test-utils/index.js';
22
import type { TestProject } from '../src/test-utils/index.js';
3+
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
34
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
45

6+
const telemetry = createTelemetryHelper();
7+
58
describe('integration: add and remove resources', () => {
69
let project: TestProject;
710

@@ -16,13 +19,16 @@ describe('integration: add and remove resources', () => {
1619

1720
afterAll(async () => {
1821
await project.cleanup();
22+
telemetry.destroy();
1923
});
2024

2125
describe('memory lifecycle', () => {
2226
const memoryName = `IntegMem${Date.now().toString().slice(-6)}`;
2327

2428
it('adds a memory resource', async () => {
25-
const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath);
29+
const result = await runCLI(['add', 'memory', '--name', memoryName, '--json'], project.projectPath, {
30+
env: telemetry.env,
31+
});
2632

2733
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
2834
const json = JSON.parse(result.stdout);
@@ -34,13 +40,17 @@ describe('integration: add and remove resources', () => {
3440
expect(memories, 'memories should exist').toBeDefined();
3541
const found = memories!.some((m: Record<string, unknown>) => m.name === memoryName);
3642
expect(found, `Memory "${memoryName}" should be in config`).toBe(true);
43+
44+
// Verify telemetry
45+
telemetry.assertMetricEmitted({ command: 'add.memory', exit_reason: 'success' });
3746
});
3847

3948
it('adds a memory with EPISODIC strategy and verifies reflectionNamespaces', async () => {
4049
const episodicMemName = `EpiMem${Date.now().toString().slice(-6)}`;
4150
const result = await runCLI(
4251
['add', 'memory', '--name', episodicMemName, '--strategies', 'EPISODIC', '--json'],
43-
project.projectPath
52+
project.projectPath,
53+
{ env: telemetry.env }
4454
);
4555

4656
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
@@ -61,6 +71,14 @@ describe('integration: add and remove resources', () => {
6171
expect(episodic!.reflectionNamespaces, 'Should have reflectionNamespaces').toBeDefined();
6272
expect(episodic!.reflectionNamespaces!.length).toBeGreaterThan(0);
6373

74+
// Verify telemetry
75+
telemetry.assertMetricEmitted({
76+
command: 'add.memory',
77+
exit_reason: 'success',
78+
strategy_count: '1',
79+
strategy_episodic: 'true',
80+
});
81+
6482
// Clean up
6583
await runCLI(['remove', 'memory', '--name', episodicMemName, '--json'], project.projectPath);
6684
});
@@ -86,7 +104,8 @@ describe('integration: add and remove resources', () => {
86104
it('adds a credential resource', async () => {
87105
const result = await runCLI(
88106
['add', 'credential', '--name', credentialName, '--api-key', 'test-key-integ-123', '--json'],
89-
project.projectPath
107+
project.projectPath,
108+
{ env: telemetry.env }
90109
);
91110

92111
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
@@ -99,6 +118,13 @@ describe('integration: add and remove resources', () => {
99118
expect(credentials, 'credentials should exist').toBeDefined();
100119
const found = credentials!.some((c: Record<string, unknown>) => c.name === credentialName);
101120
expect(found, `Credential "${credentialName}" should be in config`).toBe(true);
121+
122+
// Verify telemetry
123+
telemetry.assertMetricEmitted({
124+
command: 'add.credential',
125+
exit_reason: 'success',
126+
credential_type: 'api-key',
127+
});
102128
});
103129

104130
it('removes the credential resource', async () => {
@@ -115,4 +141,30 @@ describe('integration: add and remove resources', () => {
115141
expect(found, `Credential "${credentialName}" should be removed from config`).toBe(false);
116142
});
117143
});
144+
145+
describe('policy-engine', () => {
146+
const engineName = `TestEngine${Date.now().toString().slice(-6)}`;
147+
148+
it('adds a policy engine resource', async () => {
149+
const result = await runCLI(['add', 'policy-engine', '--name', engineName, '--json'], project.projectPath, {
150+
env: telemetry.env,
151+
});
152+
153+
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
154+
const json = JSON.parse(result.stdout);
155+
expect(json.success).toBe(true);
156+
157+
telemetry.assertMetricEmitted({
158+
command: 'add.policy-engine',
159+
exit_reason: 'success',
160+
attach_gateway_count: '0',
161+
});
162+
});
163+
164+
it('removes the policy engine resource', async () => {
165+
const result = await runCLI(['remove', 'policy-engine', '--name', engineName, '--json'], project.projectPath);
166+
167+
expect(result.exitCode, `stdout: ${result.stdout}, stderr: ${result.stderr}`).toBe(0);
168+
});
169+
});
118170
});

‎integ-tests/create-no-agent.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ describe('integration: create without agent', () => {
3232

3333
it.skipIf(!hasNpm || !hasGit)('creates project with real npm install and git init', async () => {
3434
const name = `NoAgent${Date.now().toString().slice(-6)}`;
35-
const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, false);
35+
const result = await runCLI(['create', '--name', name, '--no-agent', '--json'], testDir, { skipInstall: false });
3636

3737
expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
3838

‎integ-tests/create-with-agent.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ describe('integration: create with Python agent', () => {
4949
'--json',
5050
],
5151
testDir,
52-
false
52+
{ skipInstall: false }
5353
);
5454

5555
expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);

‎integ-tests/dev-server.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ describe('integration: dev server', () => {
6060
'--json',
6161
],
6262
testDir,
63-
false
63+
{ skipInstall: false }
6464
);
6565

6666
if (result.exitCode === 0) {

‎integ-tests/help.test.ts‎

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
import { spawnAndCollect } from '../src/test-utils/cli-runner.js';
22
import { runCLI } from '../src/test-utils/index.js';
3+
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
34
import { readdirSync } from 'node:fs';
4-
import { mkdir, readFile, rm } from 'node:fs/promises';
5-
import { tmpdir } from 'node:os';
65
import { join } from 'node:path';
7-
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
6+
import { afterAll, describe, expect, it } from 'vitest';
87

98
const COMMANDS = [
109
'create',
@@ -45,52 +44,46 @@ describe('CLI help', () => {
4544
});
4645

4746
describe('help modes telemetry', () => {
48-
let testConfigDir: string;
47+
const telemetry = createTelemetryHelper();
4948
const cliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs');
5049

51-
beforeAll(async () => {
52-
testConfigDir = join(tmpdir(), `agentcore-help-telemetry-${Date.now()}`);
53-
await mkdir(testConfigDir, { recursive: true });
54-
});
55-
afterAll(() => rm(testConfigDir, { recursive: true, force: true }));
50+
afterAll(() => telemetry.destroy());
5651

5752
function run(args: string[], extraEnv: Record<string, string> = {}) {
58-
return spawnAndCollect('node', [cliPath, ...args], tmpdir(), {
53+
return spawnAndCollect('node', [cliPath, ...args], process.cwd(), {
5954
AGENTCORE_SKIP_INSTALL: '1',
60-
AGENTCORE_CONFIG_DIR: testConfigDir,
55+
...telemetry.env,
6156
...extraEnv,
6257
});
6358
}
6459

6560
it('writes JSONL audit file when audit is enabled via env var', async () => {
66-
const result = await run(['help', 'modes'], { AGENTCORE_TELEMETRY_AUDIT: '1' });
61+
const result = await run(['help', 'modes']);
6762
expect(result.exitCode).toBe(0);
6863

69-
const telemetryDir = join(testConfigDir, 'telemetry');
70-
const files = readdirSync(telemetryDir).filter(f => f.startsWith('help-'));
71-
expect(files).toHaveLength(1);
72-
73-
const content = await readFile(join(telemetryDir, files[0]!), 'utf-8');
74-
const entry = JSON.parse(content.trim());
75-
expect(entry.attrs).toMatchObject({
76-
'service.name': 'agentcore-cli',
77-
'agentcore-cli.mode': 'cli',
64+
const entries = telemetry.readEntries();
65+
expect(entries).toHaveLength(1);
66+
telemetry.assertMetricEmitted({
7867
command_group: 'help',
7968
command: 'help.modes',
8069
exit_reason: 'success',
8170
});
82-
expect(entry.attrs['agentcore-cli.session_id']).toBeDefined();
83-
expect(entry.attrs['os.type']).toBeDefined();
84-
expect(entry.value).toBeGreaterThanOrEqual(0);
71+
expect(entries[0]!.attrs['agentcore-cli.session_id']).toBeDefined();
72+
expect(entries[0]!.attrs['os.type']).toBeDefined();
73+
expect(entries[0]!.value).toBeGreaterThanOrEqual(0);
8574
});
8675

8776
it('does not write audit file when audit is not enabled', async () => {
88-
const telemetryDir = join(testConfigDir, 'telemetry');
89-
await rm(telemetryDir, { recursive: true, force: true });
77+
telemetry.clearEntries();
9078

91-
const result = await run(['help', 'modes']);
79+
const noAuditCliPath = join(__dirname, '..', 'dist', 'cli', 'index.mjs');
80+
const result = await spawnAndCollect('node', [noAuditCliPath, 'help', 'modes'], process.cwd(), {
81+
AGENTCORE_SKIP_INSTALL: '1',
82+
AGENTCORE_CONFIG_DIR: telemetry.dir,
83+
});
9284
expect(result.exitCode).toBe(0);
9385

86+
const telemetryDir = join(telemetry.dir, 'telemetry');
9487
try {
9588
const files = readdirSync(telemetryDir);
9689
expect(files).toHaveLength(0);

0 commit comments

Comments
 (0)