diff --git a/src/assets/cdk/package.json b/src/assets/cdk/package.json index aa58892c2..0c6270f5f 100644 --- a/src/assets/cdk/package.json +++ b/src/assets/cdk/package.json @@ -23,7 +23,7 @@ "typescript": "~5.9.3" }, "dependencies": { - "@aws/agentcore-cdk": "^0.1.0-alpha.19", + "@aws/agentcore-cdk": "^0.1.0-alpha.29", "aws-cdk-lib": "^2.248.0", "constructs": "^10.0.0" } diff --git a/src/cli/aws/agentcore-control.ts b/src/cli/aws/agentcore-control.ts index 162c2b3a6..549ed134e 100644 --- a/src/cli/aws/agentcore-control.ts +++ b/src/cli/aws/agentcore-control.ts @@ -740,6 +740,10 @@ export interface GetOnlineEvalConfigResult { outputLogGroupName?: string; /** Sampling percentage from the rule config */ samplingPercentage?: number; + /** Session idle timeout in minutes from rule.sessionConfig (if set) */ + sessionTimeoutMinutes?: number; + /** Filter rules from rule.filters (if any) */ + filters?: import('../../schema').FilterRule[]; /** Service names from CloudWatch data source config (e.g. "projectName_agentName.DEFAULT") */ serviceNames?: string[]; /** Evaluator IDs referenced by this config */ @@ -763,6 +767,19 @@ export async function getOnlineEvaluationConfig( const logGroupName = response.outputConfig?.cloudWatchConfig?.logGroupName; const samplingPercentage = response.rule?.samplingConfig?.samplingPercentage; + const sessionTimeoutMinutes = response.rule?.sessionConfig?.sessionTimeoutMinutes; + const filters = (response.rule?.filters ?? []) + .map(f => { + if (!f.key || !f.operator || !f.value) return undefined; + const v = f.value; + const value: { stringValue?: string; doubleValue?: number; booleanValue?: boolean } = {}; + if ('stringValue' in v && v.stringValue !== undefined) value.stringValue = v.stringValue; + else if ('doubleValue' in v && v.doubleValue !== undefined) value.doubleValue = v.doubleValue; + else if ('booleanValue' in v && v.booleanValue !== undefined) value.booleanValue = v.booleanValue; + else return undefined; // unknown variant — skip + return { key: f.key, operator: f.operator as import('../../schema').FilterOperator, value }; + }) + .filter((f): f is import('../../schema').FilterRule => f !== undefined); const serviceNames = response.dataSourceConfig && 'cloudWatchLogs' in response.dataSourceConfig ? response.dataSourceConfig.cloudWatchLogs?.serviceNames @@ -781,6 +798,8 @@ export async function getOnlineEvaluationConfig( failureReason: response.failureReason, outputLogGroupName: logGroupName, samplingPercentage, + ...(sessionTimeoutMinutes !== undefined && { sessionTimeoutMinutes }), + ...(filters.length > 0 && { filters }), serviceNames, evaluatorIds, }; diff --git a/src/cli/commands/import/__tests__/import-online-eval.test.ts b/src/cli/commands/import/__tests__/import-online-eval.test.ts index 57f0137fc..bb1dfacd9 100644 --- a/src/cli/commands/import/__tests__/import-online-eval.test.ts +++ b/src/cli/commands/import/__tests__/import-online-eval.test.ts @@ -106,6 +106,50 @@ describe('toOnlineEvalConfigSpec', () => { expect(result.description).toBeUndefined(); }); + it('preserves sessionTimeoutMinutes and filters when present', () => { + const detail: GetOnlineEvalConfigResult = { + configId: 'oec-rich', + configArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/oec-rich', + configName: 'Rich', + status: 'ACTIVE', + executionStatus: 'ENABLED', + samplingPercentage: 30, + sessionTimeoutMinutes: 60, + filters: [ + { key: 'userId', operator: 'Equals', value: { stringValue: 'abc' } }, + { key: 'score', operator: 'GreaterThan', value: { doubleValue: 0.5 } }, + ], + serviceNames: ['agent.DEFAULT'], + evaluatorIds: ['eval-1'], + }; + + const result = toOnlineEvalConfigSpec(detail, 'Rich', 'agent', ['eval_one']); + + expect(result.sessionTimeoutMinutes).toBe(60); + expect(result.filters).toEqual([ + { key: 'userId', operator: 'Equals', value: { stringValue: 'abc' } }, + { key: 'score', operator: 'GreaterThan', value: { doubleValue: 0.5 } }, + ]); + }); + + it('omits sessionTimeoutMinutes and filters when absent on the source config', () => { + const detail: GetOnlineEvalConfigResult = { + configId: 'oec-bare', + configArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:online-evaluation-config/oec-bare', + configName: 'Bare', + status: 'ACTIVE', + executionStatus: 'ENABLED', + samplingPercentage: 5, + serviceNames: ['agent.DEFAULT'], + evaluatorIds: ['eval-1'], + }; + + const result = toOnlineEvalConfigSpec(detail, 'Bare', 'agent', ['eval_one']); + + expect(result.sessionTimeoutMinutes).toBeUndefined(); + expect(result.filters).toBeUndefined(); + }); + it('throws when sampling percentage is missing', () => { const detail: GetOnlineEvalConfigResult = { configId: 'oec-no-sampling', diff --git a/src/cli/commands/import/import-online-eval.ts b/src/cli/commands/import/import-online-eval.ts index 99935ecdf..2cc3c1be8 100644 --- a/src/cli/commands/import/import-online-eval.ts +++ b/src/cli/commands/import/import-online-eval.ts @@ -44,6 +44,8 @@ export function toOnlineEvalConfigSpec( samplingRate: detail.samplingPercentage, ...(detail.description && { description: detail.description }), ...(detail.executionStatus === 'ENABLED' && { enableOnCreate: true }), + ...(detail.sessionTimeoutMinutes !== undefined && { sessionTimeoutMinutes: detail.sessionTimeoutMinutes }), + ...(detail.filters && detail.filters.length > 0 && { filters: detail.filters }), }; } diff --git a/src/cli/operations/dev/web-ui/api-types.ts b/src/cli/operations/dev/web-ui/api-types.ts index 8ba57937e..75f7df4d3 100644 --- a/src/cli/operations/dev/web-ui/api-types.ts +++ b/src/cli/operations/dev/web-ui/api-types.ts @@ -201,6 +201,14 @@ export interface ResourceOnlineEvalConfig { evaluators: string[]; samplingRate: number; description?: string; + /** Session idle timeout in minutes (1-1440). Default: 5 */ + sessionTimeoutMinutes?: number; + /** Optional filter rules limiting which traces are evaluated. */ + filters?: Array<{ + key: string; + operator: string; + value: { stringValue?: string; doubleValue?: number; booleanValue?: boolean }; + }>; deploymentStatus?: ResourceDeploymentStatus; deployed?: DeployedOnlineEvalState; } diff --git a/src/cli/primitives/OnlineEvalConfigPrimitive.ts b/src/cli/primitives/OnlineEvalConfigPrimitive.ts index 6e4436ac9..b600ca1a7 100644 --- a/src/cli/primitives/OnlineEvalConfigPrimitive.ts +++ b/src/cli/primitives/OnlineEvalConfigPrimitive.ts @@ -1,11 +1,12 @@ import { findConfigRoot } from '../../lib'; -import type { OnlineEvalConfig } from '../../schema'; +import type { FilterRule, OnlineEvalConfig } from '../../schema'; import { OnlineEvalConfigSchema } from '../../schema'; import { getErrorMessage } from '../errors'; import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types'; import { cliCommandRun } from '../telemetry/cli-command-run.js'; import { requireTTY } from '../tui/guards/tty'; import { BasePrimitive } from './BasePrimitive'; +import { parseFilterFlags } from './filter-flag-parser'; import type { AddResult, AddScreenComponent, RemovableResource } from './types'; import type { Command } from '@commander-js/extra-typings'; @@ -16,6 +17,8 @@ export interface AddOnlineEvalConfigOptions { samplingRate: number; enableOnCreate?: boolean; endpoint?: string; + sessionTimeoutMinutes?: number; + filters?: FilterRule[]; } export type RemovableOnlineEvalConfig = RemovableResource; @@ -113,6 +116,11 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive', 'Sampling percentage (0.01-100) [non-interactive]') .option('--endpoint ', 'Runtime endpoint name to scope monitoring [non-interactive]') .option('--enable-on-create', 'Enable evaluation immediately after deploy [non-interactive]') + .option('--session-timeout-minutes ', 'Session idle timeout in minutes (1-1440) [non-interactive]') + .option( + '--filter ', + 'Filter rule (repeatable). Format: key=,op=,type=,value= [non-interactive]' + ) .option('--json', 'Output as JSON [non-interactive]') .action( async (cliOptions: { @@ -123,6 +131,8 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive { if (!findConfigRoot()) { @@ -149,6 +159,19 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive 1440) { + throw new Error( + `Invalid --session-timeout-minutes "${cliOptions.sessionTimeoutMinutes}". Must be an integer between 1 and 1440` + ); + } + sessionTimeoutMinutes = parsed; + } + + const filters = parseFilterFlags(cliOptions.filter); + const result = await this.add({ name: cliOptions.name, agent: cliOptions.runtime, @@ -156,6 +179,8 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive 0 && { filters }), }); if (!result.success) { @@ -171,6 +196,8 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive 0 && { filters: options.filters }), }; project.onlineEvalConfigs.push(config); diff --git a/src/cli/primitives/__tests__/OnlineEvalConfigPrimitive.test.ts b/src/cli/primitives/__tests__/OnlineEvalConfigPrimitive.test.ts index c81160a6c..cadcd7b9a 100644 --- a/src/cli/primitives/__tests__/OnlineEvalConfigPrimitive.test.ts +++ b/src/cli/primitives/__tests__/OnlineEvalConfigPrimitive.test.ts @@ -95,6 +95,76 @@ describe('OnlineEvalConfigPrimitive', () => { expect(config.enableOnCreate).toBeUndefined(); }); + it('persists sessionTimeoutMinutes when provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const result = await primitive.add({ + name: 'WithTimeout', + agent: 'MyAgent', + evaluators: ['Builtin.GoalSuccessRate'], + samplingRate: 10, + sessionTimeoutMinutes: 30, + }); + + expect(result.success).toBe(true); + const config = mockWriteProjectSpec.mock.calls[0]![0].onlineEvalConfigs[0]; + expect(config.sessionTimeoutMinutes).toBe(30); + }); + + it('omits sessionTimeoutMinutes when not provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'NoTimeout', + agent: 'MyAgent', + evaluators: ['Builtin.GoalSuccessRate'], + samplingRate: 10, + }); + + const config = mockWriteProjectSpec.mock.calls[0]![0].onlineEvalConfigs[0]; + expect(config.sessionTimeoutMinutes).toBeUndefined(); + }); + + it('persists filters when provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + const filters = [ + { key: 'userId', operator: 'Equals' as const, value: { stringValue: 'abc' } }, + { key: 'score', operator: 'GreaterThan' as const, value: { doubleValue: 0.5 } }, + ]; + + const result = await primitive.add({ + name: 'WithFilters', + agent: 'MyAgent', + evaluators: ['Builtin.GoalSuccessRate'], + samplingRate: 10, + filters, + }); + + expect(result.success).toBe(true); + const config = mockWriteProjectSpec.mock.calls[0]![0].onlineEvalConfigs[0]; + expect(config.filters).toEqual(filters); + }); + + it('omits filters when an empty array is provided', async () => { + mockReadProjectSpec.mockResolvedValue(makeProject()); + mockWriteProjectSpec.mockResolvedValue(undefined); + + await primitive.add({ + name: 'EmptyFilters', + agent: 'MyAgent', + evaluators: ['Builtin.GoalSuccessRate'], + samplingRate: 10, + filters: [], + }); + + const config = mockWriteProjectSpec.mock.calls[0]![0].onlineEvalConfigs[0]; + expect(config.filters).toBeUndefined(); + }); + it('supports multiple evaluators including ARNs', async () => { mockReadProjectSpec.mockResolvedValue(makeProject()); mockWriteProjectSpec.mockResolvedValue(undefined); diff --git a/src/cli/primitives/__tests__/filter-flag-parser.test.ts b/src/cli/primitives/__tests__/filter-flag-parser.test.ts new file mode 100644 index 000000000..a6c50e94a --- /dev/null +++ b/src/cli/primitives/__tests__/filter-flag-parser.test.ts @@ -0,0 +1,99 @@ +import { parseFilterFlag, parseFilterFlags } from '../filter-flag-parser'; +import { describe, expect, it } from 'vitest'; + +describe('parseFilterFlag', () => { + it('parses a string filter', () => { + expect(parseFilterFlag('key=userId,op=Equals,type=string,value=abc')).toEqual({ + key: 'userId', + operator: 'Equals', + value: { stringValue: 'abc' }, + }); + }); + + it('parses a double filter', () => { + expect(parseFilterFlag('key=score,op=GreaterThan,type=double,value=0.75')).toEqual({ + key: 'score', + operator: 'GreaterThan', + value: { doubleValue: 0.75 }, + }); + }); + + it('parses a boolean filter (true)', () => { + expect(parseFilterFlag('key=isPremium,op=Equals,type=boolean,value=true')).toEqual({ + key: 'isPremium', + operator: 'Equals', + value: { booleanValue: true }, + }); + }); + + it('parses a boolean filter (false)', () => { + expect(parseFilterFlag('key=isPremium,op=NotEquals,type=boolean,value=false')).toEqual({ + key: 'isPremium', + operator: 'NotEquals', + value: { booleanValue: false }, + }); + }); + + it('keeps "true" as a string when type=string', () => { + expect(parseFilterFlag('key=k,op=Equals,type=string,value=true')).toEqual({ + key: 'k', + operator: 'Equals', + value: { stringValue: 'true' }, + }); + }); + + it('accepts fields in any order', () => { + expect(parseFilterFlag('value=abc,type=string,op=Contains,key=label')).toEqual({ + key: 'label', + operator: 'Contains', + value: { stringValue: 'abc' }, + }); + }); + + it('throws on unknown operator', () => { + expect(() => parseFilterFlag('key=k,op=StartsWith,type=string,value=x')).toThrow(/op/); + }); + + it('throws on unknown type', () => { + expect(() => parseFilterFlag('key=k,op=Equals,type=int,value=1')).toThrow(/type/); + }); + + it('throws on missing required field', () => { + expect(() => parseFilterFlag('key=k,op=Equals,type=string')).toThrow(/value/); + }); + + it('throws on duplicate keys', () => { + expect(() => parseFilterFlag('key=k,key=other,op=Equals,type=string,value=v')).toThrow(/Duplicate/); + }); + + it('throws on invalid double value', () => { + expect(() => parseFilterFlag('key=k,op=Equals,type=double,value=notanumber')).toThrow(/double/); + }); + + it('throws on invalid boolean value', () => { + expect(() => parseFilterFlag('key=k,op=Equals,type=boolean,value=yes')).toThrow(/boolean/); + }); + + it('throws on empty input', () => { + expect(() => parseFilterFlag('')).toThrow(); + expect(() => parseFilterFlag(' ')).toThrow(); + }); + + it('throws on syntactic garbage', () => { + expect(() => parseFilterFlag('not_a_kv_pair')).toThrow(); + }); +}); + +describe('parseFilterFlags', () => { + it('returns undefined for missing/empty input', () => { + expect(parseFilterFlags(undefined)).toBeUndefined(); + expect(parseFilterFlags([])).toBeUndefined(); + }); + + it('parses multiple filters', () => { + const out = parseFilterFlags(['key=a,op=Equals,type=string,value=x', 'key=b,op=GreaterThan,type=double,value=2']); + expect(out).toHaveLength(2); + expect(out![0]!.key).toBe('a'); + expect(out![1]!.value).toEqual({ doubleValue: 2 }); + }); +}); diff --git a/src/cli/primitives/filter-flag-parser.ts b/src/cli/primitives/filter-flag-parser.ts new file mode 100644 index 000000000..ecc435dcc --- /dev/null +++ b/src/cli/primitives/filter-flag-parser.ts @@ -0,0 +1,90 @@ +import type { FilterRule, FilterValue } from '../../schema'; +import { FILTER_OPERATORS, FilterRuleSchema } from '../../schema'; + +/** + * Parse a single `--filter` CLI argument into a {@link FilterRule}. + * + * DSL: `key=,op=,type=,value=` + * + * - All four keys are required. + * - Order does not matter. + * - `type=string` keeps `value` verbatim (so `value=true` stays the string `"true"`). + * - `type=double` parses with `Number(...)`. + * - `type=boolean` requires `value` to be exactly `true` or `false`. + * - Throws a descriptive Error on any malformed input. + */ +export function parseFilterFlag(raw: string): FilterRule { + if (typeof raw !== 'string' || raw.trim() === '') { + throw new Error('Empty --filter value'); + } + + const entries = new Map(); + for (const part of raw.split(',')) { + const eq = part.indexOf('='); + if (eq === -1) { + throw new Error(`Invalid --filter syntax near "${part.trim()}". Expected key=value pairs separated by commas.`); + } + const k = part.slice(0, eq).trim(); + const v = part.slice(eq + 1).trim(); + if (!k) { + throw new Error(`Invalid --filter syntax: empty key in "${raw}"`); + } + if (entries.has(k)) { + throw new Error(`Duplicate --filter key "${k}" in "${raw}"`); + } + entries.set(k, v); + } + + const required = ['key', 'op', 'type', 'value'] as const; + for (const r of required) { + if (!entries.has(r)) { + throw new Error(`--filter is missing required field "${r}". Got: "${raw}"`); + } + } + + const key = entries.get('key')!; + const op = entries.get('op')!; + const type = entries.get('type')!; + const valueStr = entries.get('value')!; + + if (!(FILTER_OPERATORS as readonly string[]).includes(op)) { + throw new Error(`Invalid --filter op "${op}". Must be one of: ${FILTER_OPERATORS.join(', ')}`); + } + + let value: FilterValue; + switch (type) { + case 'string': + value = { stringValue: valueStr }; + break; + case 'double': { + const n = Number(valueStr); + if (!Number.isFinite(n)) { + throw new Error(`Invalid --filter value "${valueStr}" for type=double`); + } + value = { doubleValue: n }; + break; + } + case 'boolean': + if (valueStr !== 'true' && valueStr !== 'false') { + throw new Error(`Invalid --filter value "${valueStr}" for type=boolean (must be "true" or "false")`); + } + value = { booleanValue: valueStr === 'true' }; + break; + default: + throw new Error(`Invalid --filter type "${type}". Must be one of: string, double, boolean`); + } + + const parsed = FilterRuleSchema.safeParse({ key, operator: op, value }); + if (!parsed.success) { + throw new Error( + `--filter failed schema validation: ${parsed.error.issues.map((i: { message: string }) => i.message).join('; ')}` + ); + } + return parsed.data; +} + +/** Parse multiple `--filter` flags. */ +export function parseFilterFlags(raws: string[] | undefined): FilterRule[] | undefined { + if (!raws || raws.length === 0) return undefined; + return raws.map(parseFilterFlag); +} diff --git a/src/cli/telemetry/schemas/command-run.ts b/src/cli/telemetry/schemas/command-run.ts index 0acfcaf1b..ff4007857 100644 --- a/src/cli/telemetry/schemas/command-run.ts +++ b/src/cli/telemetry/schemas/command-run.ts @@ -70,7 +70,12 @@ const AddCredentialAttrs = safeSchema({ credential_type: CredentialType }); const AddEvaluatorAttrs = safeSchema({ evaluator_type: EvaluatorType, level: Level }); -const AddOnlineEvalAttrs = safeSchema({ evaluator_count: Count, enable_on_create: z.boolean() }); +const AddOnlineEvalAttrs = safeSchema({ + evaluator_count: Count, + enable_on_create: z.boolean(), + filter_count: Count, + session_timeout_set: z.boolean(), +}); const AddGatewayAttrs = safeSchema({ authorizer_type: AuthorizerType, diff --git a/src/cli/tui/hooks/useCreateOnlineEval.ts b/src/cli/tui/hooks/useCreateOnlineEval.ts index b853fed05..35b3ffa16 100644 --- a/src/cli/tui/hooks/useCreateOnlineEval.ts +++ b/src/cli/tui/hooks/useCreateOnlineEval.ts @@ -1,3 +1,4 @@ +import type { FilterRule } from '../../../schema'; import { onlineEvalConfigPrimitive } from '../../primitives/registry'; import { withAddTelemetry } from '../../telemetry/cli-command-run.js'; import { useCallback, useEffect, useState } from 'react'; @@ -9,6 +10,7 @@ interface CreateOnlineEvalConfig { evaluators: string[]; samplingRate: number; sessionTimeoutMinutes?: number; + filters?: FilterRule[]; enableOnCreate: boolean; } @@ -25,6 +27,8 @@ export function useCreateOnlineEval() { { evaluator_count: config.evaluators.length, enable_on_create: config.enableOnCreate ?? false, + filter_count: config.filters?.length ?? 0, + session_timeout_set: config.sessionTimeoutMinutes !== undefined, }, () => onlineEvalConfigPrimitive.add({ @@ -34,6 +38,7 @@ export function useCreateOnlineEval() { evaluators: config.evaluators, samplingRate: config.samplingRate, ...(config.sessionTimeoutMinutes !== undefined && { sessionTimeoutMinutes: config.sessionTimeoutMinutes }), + ...(config.filters && config.filters.length > 0 && { filters: config.filters }), enableOnCreate: config.enableOnCreate, }) ); diff --git a/src/cli/tui/screens/online-eval/AddOnlineEvalScreen.tsx b/src/cli/tui/screens/online-eval/AddOnlineEvalScreen.tsx index fd5fafcf6..19f883670 100644 --- a/src/cli/tui/screens/online-eval/AddOnlineEvalScreen.tsx +++ b/src/cli/tui/screens/online-eval/AddOnlineEvalScreen.tsx @@ -12,6 +12,7 @@ import { import { HELP_TEXT } from '../../constants'; import { useListNavigation, useMultiSelectNavigation } from '../../hooks'; import { generateUniqueName } from '../../utils'; +import { FilterBuilder } from './FilterBuilder'; import type { AddOnlineEvalConfig, EvaluatorItem, RuntimeEndpointEntry } from './types'; import { DEFAULT_SAMPLING_RATE, ONLINE_EVAL_STEP_LABELS } from './types'; import { useAddOnlineEvalWizard } from './useAddOnlineEvalWizard'; @@ -99,6 +100,8 @@ export function AddOnlineEvalScreen({ const isEndpointStep = wizard.step === 'endpoint'; const isEvaluatorsStep = wizard.step === 'evaluators'; const isSamplingRateStep = wizard.step === 'samplingRate'; + const isSessionTimeoutStep = wizard.step === 'sessionTimeoutMinutes'; + const isFiltersStep = wizard.step === 'filters'; const isEnableOnCreateStep = wizard.step === 'enableOnCreate'; const isConfirmStep = wizard.step === 'confirm'; @@ -152,7 +155,7 @@ export function AddOnlineEvalScreen({ const helpText = isEvaluatorsStep ? 'Space toggle · Enter confirm · Esc back' - : isAgentStep || isEndpointStep || isEnableOnCreateStep + : isAgentStep || isEndpointStep || isEnableOnCreateStep || isFiltersStep ? HELP_TEXT.NAVIGATE_SELECT : isConfirmStep ? HELP_TEXT.CONFIRM_CANCEL @@ -230,6 +233,47 @@ export function AddOnlineEvalScreen({ )} + {isSessionTimeoutStep && ( + + + How long after the last trace a session is considered complete (1–1440 minutes). Leave blank to use the + default of 5 minutes. + + { + const trimmed = value.trim(); + if (trimmed === '') { + wizard.setSessionTimeoutMinutes(undefined); + return; + } + const n = Number(trimmed); + if (!Number.isInteger(n) || n < 1 || n > 1440) return; + wizard.setSessionTimeoutMinutes(n); + }} + onCancel={() => wizard.goBack()} + customValidation={value => { + const trimmed = value.trim(); + if (trimmed === '') return true; + const n = Number(trimmed); + if (!Number.isInteger(n)) return 'Must be a whole number (or blank)'; + if (n < 1 || n > 1440) return 'Must be between 1 and 1440'; + return true; + }} + /> + + )} + + {isFiltersStep && ( + wizard.setFilters(filters.length > 0 ? filters : undefined)} + onCancel={() => wizard.goBack()} + /> + )} + {isEnableOnCreateStep && ( 0 + ? effectiveConfig.filters.map(f => `${f.key} ${f.operator} ${JSON.stringify(f.value)}`).join('; ') + : 'None', + }, { label: 'Enable on Deploy', value: effectiveConfig.enableOnCreate ? 'Yes' : 'No' }, ]} /> diff --git a/src/cli/tui/screens/online-eval/FilterBuilder.tsx b/src/cli/tui/screens/online-eval/FilterBuilder.tsx new file mode 100644 index 000000000..d27c7235e --- /dev/null +++ b/src/cli/tui/screens/online-eval/FilterBuilder.tsx @@ -0,0 +1,195 @@ +import type { FilterRule, FilterValue } from '../../../../schema'; +import type { SelectableItem } from '../../components'; +import { ConfirmReview, Panel, TextInput, WizardSelect } from '../../components'; +import { useListNavigation } from '../../hooks'; +import { FILTER_OPERATOR_OPTIONS, type FilterValueType } from './types'; +import { Box, Text } from 'ink'; +import React, { useCallback, useMemo, useState } from 'react'; + +type SubStep = + | 'start' + | 'enter-key' + | 'pick-operator' + | 'pick-value-type' + | 'enter-value' + | 'pick-bool-value' + | 'review'; + +interface FilterBuilderProps { + initial?: FilterRule[]; + onComplete: (filters: FilterRule[]) => void; + onCancel: () => void; +} + +/** + * Inline filter sub-wizard. Lets the user build zero or more {@link FilterRule}s, + * one at a time, and confirm. + */ +export function FilterBuilder({ initial, onComplete, onCancel }: FilterBuilderProps) { + const [filters, setFilters] = useState(initial ?? []); + const [sub, setSub] = useState('start'); + const [draftKey, setDraftKey] = useState(''); + const [draftOp, setDraftOp] = useState<(typeof FILTER_OPERATOR_OPTIONS)[number]>('Equals'); + const [draftType, setDraftType] = useState('string'); + + const finishDraft = useCallback( + (value: FilterValue) => { + setFilters(prev => [...prev, { key: draftKey, operator: draftOp, value }]); + setDraftKey(''); + setDraftOp('Equals'); + setDraftType('string'); + setSub('start'); + }, + [draftKey, draftOp] + ); + + const startItems: SelectableItem[] = useMemo( + () => [ + { id: 'add', title: filters.length === 0 ? 'Add a filter' : 'Add another filter' }, + { id: 'done', title: filters.length === 0 ? 'No filters (skip)' : 'Done' }, + ...(filters.length > 0 + ? [{ id: 'clear', title: 'Remove last filter', description: filters[filters.length - 1]!.key }] + : []), + ], + [filters] + ); + + const startNav = useListNavigation({ + items: startItems, + onSelect: item => { + if (item.id === 'add') setSub('enter-key'); + else if (item.id === 'done') onComplete(filters); + else if (item.id === 'clear') setFilters(prev => prev.slice(0, -1)); + }, + onExit: () => onCancel(), + isActive: sub === 'start', + }); + + const operatorItems: SelectableItem[] = useMemo(() => FILTER_OPERATOR_OPTIONS.map(op => ({ id: op, title: op })), []); + const operatorNav = useListNavigation({ + items: operatorItems, + onSelect: item => { + setDraftOp(item.id as (typeof FILTER_OPERATOR_OPTIONS)[number]); + setSub('pick-value-type'); + }, + onExit: () => setSub('enter-key'), + isActive: sub === 'pick-operator', + }); + + const typeItems: SelectableItem[] = useMemo( + () => [ + { id: 'string', title: 'string' }, + { id: 'double', title: 'double (number)' }, + { id: 'boolean', title: 'boolean' }, + ], + [] + ); + const typeNav = useListNavigation({ + items: typeItems, + onSelect: item => { + setDraftType(item.id as FilterValueType); + setSub(item.id === 'boolean' ? 'pick-bool-value' : 'enter-value'); + }, + onExit: () => setSub('pick-operator'), + isActive: sub === 'pick-value-type', + }); + + const boolItems: SelectableItem[] = useMemo( + () => [ + { id: 'true', title: 'true' }, + { id: 'false', title: 'false' }, + ], + [] + ); + const boolNav = useListNavigation({ + items: boolItems, + onSelect: item => finishDraft({ booleanValue: item.id === 'true' }), + onExit: () => setSub('pick-value-type'), + isActive: sub === 'pick-bool-value', + }); + + return ( + + {sub === 'start' && ( + + + Optional: filter rules. Only traces matching every filter are evaluated. Leave empty to evaluate all sampled + traces. + + {filters.length > 0 && ( + + Current filters: + {filters.map((f, i) => ( + + {` [${i + 1}] ${f.key} ${f.operator} ${JSON.stringify(f.value)}`} + + ))} + + )} + + + + + )} + + {sub === 'enter-key' && ( + { + setDraftKey(v); + setSub('pick-operator'); + }} + onCancel={() => setSub('start')} + customValidation={v => (v.trim().length > 0 ? true : 'Key cannot be empty')} + /> + )} + + {sub === 'pick-operator' && ( + + )} + + {sub === 'pick-value-type' && ( + + )} + + {sub === 'enter-value' && draftType !== 'boolean' && ( + { + if (draftType === 'double') { + const n = Number(v); + if (!Number.isFinite(n)) return; + finishDraft({ doubleValue: n }); + } else { + finishDraft({ stringValue: v }); + } + }} + onCancel={() => setSub('pick-value-type')} + customValidation={v => { + if (draftType === 'double') { + return Number.isFinite(Number(v)) || 'Must be a number'; + } + return v.length > 0 || 'Value cannot be empty'; + }} + /> + )} + + {sub === 'pick-bool-value' && ( + + )} + + {sub === 'review' && ( + ({ + label: `Filter ${i + 1}`, + value: `${f.key} ${f.operator} ${JSON.stringify(f.value)}`, + }))} + /> + )} + + ); +} diff --git a/src/cli/tui/screens/online-eval/types.ts b/src/cli/tui/screens/online-eval/types.ts index 1a1e5940c..12b044de8 100644 --- a/src/cli/tui/screens/online-eval/types.ts +++ b/src/cli/tui/screens/online-eval/types.ts @@ -1,6 +1,8 @@ -// ───────────────────────────────────────────────────────────────────────────── +import type { FilterRule } from '../../../../schema'; + +// ──────────────────────────────────────────────────────────────────────────── // Online Eval Config Flow Types -// ───────────────────────────────────────────────────────────────────────────── +// ──────────────────────────────────────────────────────────────────────────── export type AddOnlineEvalStep = | 'name' @@ -8,6 +10,8 @@ export type AddOnlineEvalStep = | 'endpoint' | 'evaluators' | 'samplingRate' + | 'sessionTimeoutMinutes' + | 'filters' | 'enableOnCreate' | 'confirm'; @@ -17,6 +21,10 @@ export interface AddOnlineEvalConfig { endpoint?: string; evaluators: string[]; samplingRate: number; + /** Session idle timeout in minutes (1-1440). Undefined = use service/CDK default of 5. */ + sessionTimeoutMinutes?: number; + /** Optional list of trace-level filters (max 20). */ + filters?: FilterRule[]; enableOnCreate: boolean; description?: string; } @@ -33,13 +41,32 @@ export const ONLINE_EVAL_STEP_LABELS: Record = { endpoint: 'Endpoint', evaluators: 'Evaluators', samplingRate: 'Rate', + sessionTimeoutMinutes: 'Timeout', + filters: 'Filters', enableOnCreate: 'Enable', confirm: 'Confirm', }; -// ───────────────────────────────────────────────────────────────────────────── +// ──────────────────────────────────────────────────────────────────────────── +// Filter constants for the TUI builder +// ──────────────────────────────────────────────────────────────────────────── + +export const FILTER_OPERATOR_OPTIONS = [ + 'Equals', + 'NotEquals', + 'GreaterThan', + 'LessThan', + 'GreaterThanOrEqual', + 'LessThanOrEqual', + 'Contains', + 'NotContains', +] as const; + +export type FilterValueType = 'string' | 'double' | 'boolean'; + +// ──────────────────────────────────────────────────────────────────────────── // Evaluator Items (fetched from API) -// ───────────────────────────────────────────────────────────────────────────── +// ──────────────────────────────────────────────────────────────────────────── export interface EvaluatorItem { /** ARN used as the stored identifier in the config */ diff --git a/src/cli/tui/screens/online-eval/useAddOnlineEvalWizard.ts b/src/cli/tui/screens/online-eval/useAddOnlineEvalWizard.ts index 239a95edc..e528c87b5 100644 --- a/src/cli/tui/screens/online-eval/useAddOnlineEvalWizard.ts +++ b/src/cli/tui/screens/online-eval/useAddOnlineEvalWizard.ts @@ -1,3 +1,4 @@ +import type { FilterRule } from '../../../../schema'; import type { AddOnlineEvalConfig, AddOnlineEvalStep } from './types'; import { DEFAULT_SAMPLING_RATE } from './types'; import { useCallback, useRef, useState } from 'react'; @@ -5,9 +6,28 @@ import { useCallback, useRef, useState } from 'react'; function getAllSteps(agentCount: number): AddOnlineEvalStep[] { if (agentCount <= 1) { // endpoint step is included but will be skipped dynamically when no endpoints exist - return ['name', 'endpoint', 'evaluators', 'samplingRate', 'enableOnCreate', 'confirm']; + return [ + 'name', + 'endpoint', + 'evaluators', + 'samplingRate', + 'sessionTimeoutMinutes', + 'filters', + 'enableOnCreate', + 'confirm', + ]; } - return ['name', 'agent', 'endpoint', 'evaluators', 'samplingRate', 'enableOnCreate', 'confirm']; + return [ + 'name', + 'agent', + 'endpoint', + 'evaluators', + 'samplingRate', + 'sessionTimeoutMinutes', + 'filters', + 'enableOnCreate', + 'confirm', + ]; } function getDefaultConfig(): AddOnlineEvalConfig { @@ -17,6 +37,8 @@ function getDefaultConfig(): AddOnlineEvalConfig { endpoint: undefined, evaluators: [], samplingRate: DEFAULT_SAMPLING_RATE, + sessionTimeoutMinutes: undefined, + filters: undefined, enableOnCreate: true, }; } @@ -102,6 +124,25 @@ export function useAddOnlineEvalWizard(agentCount: number) { [nextStep, setConfig, setStep] ); + const setSessionTimeoutMinutes = useCallback( + (sessionTimeoutMinutes: number | undefined) => { + setConfig(c => ({ ...c, sessionTimeoutMinutes })); + const next = nextStep('sessionTimeoutMinutes'); + if (next) setStep(next); + }, + [nextStep, setConfig, setStep] + ); + + const setFilters = useCallback( + (filters: FilterRule[] | undefined) => { + const cleaned = filters && filters.length > 0 ? filters : undefined; + setConfig(c => ({ ...c, filters: cleaned })); + const next = nextStep('filters'); + if (next) setStep(next); + }, + [nextStep, setConfig, setStep] + ); + const setEnableOnCreate = useCallback( (enableOnCreate: boolean) => { setConfig(c => ({ ...c, enableOnCreate })); @@ -128,6 +169,8 @@ export function useAddOnlineEvalWizard(agentCount: number) { setEndpoint, setEvaluators, setSamplingRate, + setSessionTimeoutMinutes, + setFilters, setEnableOnCreate, reset, }; diff --git a/src/schema/llm-compacted/agentcore.ts b/src/schema/llm-compacted/agentcore.ts index c86c14cb7..ae3954ce1 100644 --- a/src/schema/llm-compacted/agentcore.ts +++ b/src/schema/llm-compacted/agentcore.ts @@ -176,10 +176,26 @@ interface OnlineEvalConfig { evaluators: string[]; // @min 1 — evaluator names, Builtin.* IDs, or evaluator ARNs samplingRate: number; // @min 0.01 @max 100 (percentage) description?: string; // @max 200 + sessionTimeoutMinutes?: number; // @min 1 @max 1440 — session idle timeout. Default: 5 + filters?: FilterRule[]; // Optional trace-level filters (max 20) enableOnCreate?: boolean; // Whether to enable on create (default: true) tags?: Record; } +interface FilterRule { + key: string; + operator: + | 'Equals' + | 'NotEquals' + | 'GreaterThan' + | 'LessThan' + | 'GreaterThanOrEqual' + | 'LessThanOrEqual' + | 'Contains' + | 'NotContains'; + value: { stringValue?: string; doubleValue?: number; booleanValue?: boolean }; // exactly one variant +} + // ───────────────────────────────────────────────────────────────────────────── // GATEWAY (MCP) // ───────────────────────────────────────────────────────────────────────────── diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts index 10d164a2c..ef1ec701d 100644 --- a/src/schema/schemas/agentcore-project.ts +++ b/src/schema/schemas/agentcore-project.ts @@ -34,8 +34,15 @@ export { }; export { EvaluationLevelSchema }; export type { MemoryStrategy, MemoryStrategyType } from './primitives/memory'; -export type { OnlineEvalConfig } from './primitives/online-eval-config'; -export { OnlineEvalConfigSchema, OnlineEvalConfigNameSchema } from './primitives/online-eval-config'; +export type { OnlineEvalConfig, FilterRule, FilterValue, FilterOperator } from './primitives/online-eval-config'; +export { + OnlineEvalConfigSchema, + OnlineEvalConfigNameSchema, + FilterRuleSchema, + FilterValueSchema, + FilterOperatorSchema, + FILTER_OPERATORS, +} from './primitives/online-eval-config'; export type { CodeBasedConfig, EvaluationLevel, diff --git a/src/schema/schemas/primitives/__tests__/online-eval-config.test.ts b/src/schema/schemas/primitives/__tests__/online-eval-config.test.ts index e6e940948..b5471fe0c 100644 --- a/src/schema/schemas/primitives/__tests__/online-eval-config.test.ts +++ b/src/schema/schemas/primitives/__tests__/online-eval-config.test.ts @@ -1,4 +1,10 @@ -import { OnlineEvalConfigNameSchema, OnlineEvalConfigSchema } from '../online-eval-config'; +import { + FILTER_OPERATORS, + FilterRuleSchema, + FilterValueSchema, + OnlineEvalConfigNameSchema, + OnlineEvalConfigSchema, +} from '../online-eval-config'; import { describe, expect, it } from 'vitest'; describe('OnlineEvalConfigNameSchema', () => { @@ -98,4 +104,83 @@ describe('OnlineEvalConfigSchema', () => { it('accepts config without description and enableOnCreate', () => { expect(OnlineEvalConfigSchema.safeParse(validConfig).success).toBe(true); }); + + it('accepts sessionTimeoutMinutes within bounds', () => { + expect(OnlineEvalConfigSchema.safeParse({ ...validConfig, sessionTimeoutMinutes: 1 }).success).toBe(true); + expect(OnlineEvalConfigSchema.safeParse({ ...validConfig, sessionTimeoutMinutes: 1440 }).success).toBe(true); + }); + + it('rejects sessionTimeoutMinutes out of bounds', () => { + expect(OnlineEvalConfigSchema.safeParse({ ...validConfig, sessionTimeoutMinutes: 0 }).success).toBe(false); + expect(OnlineEvalConfigSchema.safeParse({ ...validConfig, sessionTimeoutMinutes: 1441 }).success).toBe(false); + }); + + it('rejects non-integer sessionTimeoutMinutes', () => { + expect(OnlineEvalConfigSchema.safeParse({ ...validConfig, sessionTimeoutMinutes: 5.5 }).success).toBe(false); + }); + + it('accepts a config with filters of each value variant', () => { + const result = OnlineEvalConfigSchema.safeParse({ + ...validConfig, + filters: [ + { key: 'userId', operator: 'Equals', value: { stringValue: 'abc' } }, + { key: 'score', operator: 'GreaterThan', value: { doubleValue: 0.5 } }, + { key: 'isPremium', operator: 'Equals', value: { booleanValue: true } }, + ], + }); + expect(result.success).toBe(true); + }); + + it('rejects filters with multiple value variants', () => { + const result = OnlineEvalConfigSchema.safeParse({ + ...validConfig, + filters: [{ key: 'k', operator: 'Equals', value: { stringValue: 'a', doubleValue: 1 } }], + }); + expect(result.success).toBe(false); + }); +}); + +describe('FilterValueSchema', () => { + it('rejects empty value', () => { + expect(FilterValueSchema.safeParse({}).success).toBe(false); + }); + + it('accepts each variant alone', () => { + expect(FilterValueSchema.safeParse({ stringValue: 'x' }).success).toBe(true); + expect(FilterValueSchema.safeParse({ doubleValue: 1.2 }).success).toBe(true); + expect(FilterValueSchema.safeParse({ booleanValue: false }).success).toBe(true); + }); +}); + +describe('FilterRuleSchema', () => { + it('accepts each documented operator', () => { + for (const op of FILTER_OPERATORS) { + const result = FilterRuleSchema.safeParse({ + key: 'userId', + operator: op, + value: { stringValue: 'abc' }, + }); + expect(result.success).toBe(true); + } + }); + + it('rejects unknown operator', () => { + expect( + FilterRuleSchema.safeParse({ + key: 'userId', + operator: 'StartsWith', + value: { stringValue: 'abc' }, + }).success + ).toBe(false); + }); + + it('rejects empty key', () => { + expect( + FilterRuleSchema.safeParse({ + key: '', + operator: 'Equals', + value: { stringValue: 'abc' }, + }).success + ).toBe(false); + }); }); diff --git a/src/schema/schemas/primitives/index.ts b/src/schema/schemas/primitives/index.ts index a48985c84..39bdf7a33 100644 --- a/src/schema/schemas/primitives/index.ts +++ b/src/schema/schemas/primitives/index.ts @@ -54,8 +54,15 @@ export { RatingScaleSchema, } from './evaluator'; -export type { OnlineEvalConfig } from './online-eval-config'; -export { OnlineEvalConfigSchema, OnlineEvalConfigNameSchema } from './online-eval-config'; +export type { OnlineEvalConfig, FilterRule, FilterValue, FilterOperator } from './online-eval-config'; +export { + OnlineEvalConfigSchema, + OnlineEvalConfigNameSchema, + FilterRuleSchema, + FilterValueSchema, + FilterOperatorSchema, + FILTER_OPERATORS, +} from './online-eval-config'; export type { Policy, PolicyEngine, ValidationMode } from './policy'; export { diff --git a/src/schema/schemas/primitives/online-eval-config.ts b/src/schema/schemas/primitives/online-eval-config.ts index 5b6f13cb6..2a20d0fcc 100644 --- a/src/schema/schemas/primitives/online-eval-config.ts +++ b/src/schema/schemas/primitives/online-eval-config.ts @@ -14,6 +14,60 @@ export const OnlineEvalConfigNameSchema = z 'Must begin with a letter and contain only alphanumeric characters and underscores (max 48 chars)' ); +// ──────────────────────────────────────────────────────────────────────────── +// Filter rule types — mirror @aws/agentcore-cdk's OnlineEvaluationConfigSchema. +// Must stay in lock-step with that package; both schemas describe the same +// JSON in agentcore.json. +// ──────────────────────────────────────────────────────────────────────────── + +/** The 8 documented comparison operators for online evaluation config filters. */ +export const FILTER_OPERATORS = [ + 'Equals', + 'NotEquals', + 'GreaterThan', + 'LessThan', + 'GreaterThanOrEqual', + 'LessThanOrEqual', + 'Contains', + 'NotContains', +] as const; + +export const FilterOperatorSchema = z.enum(FILTER_OPERATORS); +export type FilterOperator = z.infer; + +/** + * Filter value — exactly one of stringValue, doubleValue, or booleanValue must + * be set (matches the boto3 / CFN spec). + */ +export const FilterValueSchema = z + .object({ + stringValue: z.string().optional(), + doubleValue: z.number().optional(), + booleanValue: z.boolean().optional(), + }) + .superRefine((val, ctx) => { + const set = [val.stringValue !== undefined, val.doubleValue !== undefined, val.booleanValue !== undefined].filter( + Boolean + ).length; + if (set !== 1) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Filter value must set exactly one of `stringValue`, `doubleValue`, or `booleanValue`', + }); + } + }); +export type FilterValue = z.infer; + +export const FilterRuleSchema = z.object({ + /** Key or field name to filter on within the agent trace data. */ + key: z.string().min(1, 'Filter key is required'), + /** Comparison operator. */ + operator: FilterOperatorSchema, + /** The value used in filter comparisons (exactly one variant). */ + value: FilterValueSchema, +}); +export type FilterRule = z.infer; + export const OnlineEvalConfigSchema = z.object({ name: OnlineEvalConfigNameSchema, /** Agent name to monitor (must match a project agent) */ @@ -26,6 +80,10 @@ export const OnlineEvalConfigSchema = z.object({ samplingRate: z.number().min(0.01).max(100), /** Optional description for the online eval config */ description: z.string().max(200).optional(), + /** Session idle timeout in minutes (1-1440). Default: 5 */ + sessionTimeoutMinutes: z.number().int().min(1).max(1440).optional(), + /** Optional list of filter rules. Only traces matching all filters are evaluated. */ + filters: z.array(FilterRuleSchema).max(20).optional(), /** Whether to enable execution on create (default: true) */ enableOnCreate: z.boolean().optional(), tags: TagsSchema.optional(),