Skip to content

Commit a079a5a

Browse files
github-actions[bot]github-actions[bot]
committed
chore: sync main with public/main
2 parents 340878c + bceddde commit a079a5a

7 files changed

Lines changed: 219 additions & 92 deletions

File tree

‎docs/PERMISSIONS.md‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,20 @@ Required for all deployment operations (`deploy`, `status`, `diff`).
335335
| `bedrock-agentcore:Evaluate` | `run evals` | Run on-demand evaluation against agent traces |
336336
| `bedrock-agentcore:UpdateOnlineEvaluationConfig` | `pause online-eval`, `resume online-eval` | Pause or resume online evaluation |
337337

338+
### Batch evaluation and recommendations
339+
340+
| Action | CLI Commands | Purpose |
341+
| ----------------------------------------- | ---------------- | ------------------------------ |
342+
| `bedrock-agentcore:StartBatchEvaluation` | `run batch-eval` | Start a batch evaluation job |
343+
| `bedrock-agentcore:GetBatchEvaluation` | `run batch-eval` | Poll batch evaluation status |
344+
| `bedrock-agentcore:ListBatchEvaluations` | `evals history` | List past batch evaluations |
345+
| `bedrock-agentcore:StopBatchEvaluation` | `run batch-eval` | Stop an in-progress batch eval |
346+
| `bedrock-agentcore:DeleteBatchEvaluation` | `run batch-eval` | Delete a batch evaluation |
347+
| `bedrock-agentcore:StartRecommendation` | `run recommend` | Start a recommendation job |
348+
| `bedrock-agentcore:GetRecommendation` | `run recommend` | Poll recommendation status |
349+
| `bedrock-agentcore:ListRecommendations` | `run recommend` | List past recommendations |
350+
| `bedrock-agentcore:DeleteRecommendation` | `run recommend` | Stop/delete a recommendation |
351+
338352
### Identity and credential management
339353

340354
| Action | CLI Commands | Purpose |
@@ -361,14 +375,19 @@ Required for all deployment operations (`deploy`, `status`, `diff`).
361375

362376
### Logging, traces, and observability
363377

364-
| Action | CLI Commands | Purpose |
365-
| ------------------------------- | ---------------------------------------- | --------------------------------------------- |
366-
| `logs:StartLiveTail` | `logs` | Stream agent logs in real-time |
367-
| `logs:FilterLogEvents` | `logs` | Search agent logs |
368-
| `logs:StartQuery` | `traces list`, `traces get`, `run evals` | Run CloudWatch Logs Insights queries |
369-
| `logs:GetQueryResults` | `traces list`, `traces get`, `run evals` | Retrieve query results |
370-
| `logs:DescribeResourcePolicies` | `deploy` | Check for X-Ray log resource policy |
371-
| `logs:PutResourcePolicy` | `deploy` | Create resource policy for X-Ray trace access |
378+
| Action | CLI Commands | Purpose |
379+
| ------------------------------- | ---------------------------------------- | ---------------------------------------------------------- |
380+
| `logs:StartLiveTail` | `logs` | Stream agent logs in real-time |
381+
| `logs:FilterLogEvents` | `logs` | Search agent logs |
382+
| `logs:StartQuery` | `traces list`, `traces get`, `run evals` | Run CloudWatch Logs Insights queries |
383+
| `logs:GetQueryResults` | `traces list`, `traces get`, `run evals` | Retrieve query results |
384+
| `logs:DescribeResourcePolicies` | `deploy` | Check for X-Ray log resource policy |
385+
| `logs:PutResourcePolicy` | `deploy` | Create resource policy for X-Ray trace access |
386+
| `logs:DescribeLogGroups` | `run batch-eval`, `run recommend` | Discover runtime log groups for evaluation data sources |
387+
| `logs:CreateLogGroup` | `run batch-eval` | Create log group for batch evaluation results output |
388+
| `logs:CreateLogStream` | `run batch-eval` | Create log stream for batch evaluation results |
389+
| `logs:PutLogEvents` | `run batch-eval` | Write batch evaluation results to CloudWatch Logs |
390+
| `logs:PutRetentionPolicy` | `run batch-eval` | Set retention policy on batch evaluation results log group |
372391

373392
### Transaction search setup
374393

‎docs/policies/iam-policy-user.json‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,22 @@
8989
"Action": ["kms:CreateKey", "kms:TagResource"],
9090
"Resource": "*"
9191
},
92+
{
93+
"Sid": "BatchEvalAndRecommendations",
94+
"Effect": "Allow",
95+
"Action": [
96+
"bedrock-agentcore:StartBatchEvaluation",
97+
"bedrock-agentcore:GetBatchEvaluation",
98+
"bedrock-agentcore:ListBatchEvaluations",
99+
"bedrock-agentcore:StopBatchEvaluation",
100+
"bedrock-agentcore:DeleteBatchEvaluation",
101+
"bedrock-agentcore:StartRecommendation",
102+
"bedrock-agentcore:GetRecommendation",
103+
"bedrock-agentcore:ListRecommendations",
104+
"bedrock-agentcore:DeleteRecommendation"
105+
],
106+
"Resource": "*"
107+
},
92108
{
93109
"Sid": "LogsStreamingAndSearch",
94110
"Effect": "Allow",
@@ -98,7 +114,12 @@
98114
"logs:StartQuery",
99115
"logs:GetQueryResults",
100116
"logs:DescribeResourcePolicies",
101-
"logs:PutResourcePolicy"
117+
"logs:PutResourcePolicy",
118+
"logs:DescribeLogGroups",
119+
"logs:CreateLogGroup",
120+
"logs:CreateLogStream",
121+
"logs:PutLogEvents",
122+
"logs:PutRetentionPolicy"
102123
],
103124
"Resource": "*"
104125
},

‎integ-tests/create-edge-cases.test.ts‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* eslint-disable security/detect-non-literal-fs-filename */
22
import { exists, prereqs, runCLI } from '../src/test-utils/index.js';
3+
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
34
import { randomUUID } from 'node:crypto';
45
import { mkdir, rm } from 'node:fs/promises';
56
import { tmpdir } from 'node:os';
@@ -9,18 +10,21 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
910
describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases', () => {
1011
let testDir: string;
1112

13+
const telemetry = createTelemetryHelper();
14+
1215
beforeAll(async () => {
1316
testDir = join(tmpdir(), `agentcore-integ-edge-${randomUUID()}`);
1417
await mkdir(testDir, { recursive: true });
1518
});
1619

1720
afterAll(async () => {
21+
telemetry.destroy();
1822
await rm(testDir, { recursive: true, force: true });
1923
});
2024

2125
describe('reserved names', () => {
2226
it('rejects reserved name "Test"', async () => {
23-
const result = await runCLI(['create', '--name', 'Test', '--json'], testDir);
27+
const result = await runCLI(['create', '--name', 'Test', '--json'], testDir, { env: telemetry.env });
2428

2529
expect(result.exitCode).toBe(1);
2630
const json = JSON.parse(result.stdout);
@@ -30,6 +34,11 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
3034
json.error.toLowerCase().includes('reserved') || json.error.toLowerCase().includes('conflict'),
3135
`Error should mention reserved/conflict: ${json.error}`
3236
).toBeTruthy();
37+
38+
telemetry.assertMetricEmitted({
39+
command: 'create',
40+
exit_reason: 'failure',
41+
});
3342
});
3443

3544
it('rejects reserved name "bedrock"', async () => {
@@ -121,12 +130,21 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create edge cases',
121130
describe('flag interactions', () => {
122131
it('--defaults creates project with default settings', async () => {
123132
const name = `Def${Date.now().toString().slice(-6)}`;
124-
const result = await runCLI(['create', '--name', name, '--defaults', '--json'], testDir);
133+
const result = await runCLI(['create', '--name', name, '--defaults', '--json'], testDir, { env: telemetry.env });
125134

126135
expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
127136
const json = JSON.parse(result.stdout);
128137
expect(json.success).toBe(true);
129138
expect(json.projectPath).toBeTruthy();
139+
140+
telemetry.assertMetricEmitted({
141+
command: 'create',
142+
exit_reason: 'success',
143+
language: 'python',
144+
framework: 'strands',
145+
model_provider: 'bedrock',
146+
has_agent: 'true',
147+
});
130148
});
131149

132150
it('--dry-run shows what would be created without writing files', async () => {

‎integ-tests/create-frameworks.test.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { exists, prereqs, readProjectConfig, runCLI } from '../src/test-utils/index.js';
2+
import { createTelemetryHelper } from '../src/test-utils/telemetry-helper.js';
23
import { randomUUID } from 'node:crypto';
34
import { mkdir, readFile, rm } from 'node:fs/promises';
45
import { tmpdir } from 'node:os';
@@ -8,12 +9,15 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest';
89
describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with different frameworks', () => {
910
let testDir: string;
1011

12+
const telemetry = createTelemetryHelper();
13+
1114
beforeAll(async () => {
1215
testDir = join(tmpdir(), `agentcore-integ-frameworks-${randomUUID()}`);
1316
await mkdir(testDir, { recursive: true });
1417
});
1518

1619
afterAll(async () => {
20+
telemetry.destroy();
1721
await rm(testDir, { recursive: true, force: true });
1822
});
1923

@@ -34,7 +38,8 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
3438
'none',
3539
'--json',
3640
],
37-
testDir
41+
testDir,
42+
{ env: telemetry.env }
3843
);
3944

4045
expect(result.exitCode, `stderr: ${result.stderr}`).toBe(0);
@@ -60,6 +65,15 @@ describe.skipIf(!prereqs.npm || !prereqs.git)('integration: create with differen
6065
expect(agents).toBeDefined();
6166
expect(agents.length).toBe(1);
6267
expect(agents[0]!.name).toBe(agentName);
68+
69+
telemetry.assertMetricEmitted({
70+
command: 'create',
71+
exit_reason: 'success',
72+
language: 'python',
73+
framework: 'langchain_langgraph',
74+
model_provider: 'bedrock',
75+
has_agent: 'true',
76+
});
6377
});
6478

6579
it('creates GoogleADK project with Gemini provider', async () => {

0 commit comments

Comments
 (0)