feat(evaluator): add kmsKeyArn support for custom evaluator (#994)

* feat(evaluator): Add kmsKeyArn support for custom evaluator

* fix: sync package-lock.json with package.json

The lock file was out of sync after dependency bumps on main were merged,
causing npm ci to fail in CI.

* fix: revert unrelated dep bumps and fix formatting

Reverts @opentelemetry/exporter-metrics-otlp-http ^0.217.0 back to
^0.214.0 and secretlint ^13.0.0 back to ^12.2.0 — these were
accidentally included in the feature commit from unmerged dependabot PRs
and introduce high-severity protobufjs vulnerabilities.

Restores fast-xml-parser and @aws-sdk/xml-builder overrides that were
also inadvertently removed.

Fixes Prettier formatting on agentcore-project.ts import lines.

* fix: sync package-lock.json with updated dependencies

---------

Co-authored-by: notgitika <gitijh@gmail.com>

Author: aws-aditya21

package-lock.json

@@ -78,7 +78,7 @@
     "@aws-sdk/client-bedrock": "^3.1012.0",
     "@aws-sdk/client-bedrock-agent": "^3.1012.0",
     "@aws-sdk/client-bedrock-agentcore": "^3.1020.0",
-    "@aws-sdk/client-bedrock-agentcore-control": "^3.1020.0",
+    "@aws-sdk/client-bedrock-agentcore-control": "^3.1039.0",
     "@aws-sdk/client-bedrock-runtime": "^3.893.0",
     "@aws-sdk/client-cloudformation": "^3.893.0",
     "@aws-sdk/client-cloudwatch-logs": "^3.893.0",
@@ -141,19 +141,23 @@
     "lint-staged": "^16.2.7",
     "node-pty": "^1.1.0",
     "prettier": "^3.7.4",
-    "secretlint": "^13.0.0",
+    "secretlint": "^12.2.0",
     "tsx": "^4.21.0",
     "typescript": "^5",
     "typescript-eslint": "^8.50.1",
     "vitest": "^4.0.18"
   },
   "overridesComments": {
     "minimatch": "GHSA-7r86-cg39-jmmj, GHSA-23c5-xmqv-rm74: minimatch 10.0.0-10.2.2 has ReDoS vulnerabilities. Multiple transitive deps (eslint, typescript-eslint, eslint-plugin-import, eslint-plugin-react, prettier-plugin-sort-imports, aws-cdk-lib) pin older versions. Remove this override once upstream packages update their minimatch dependency to >=10.2.3.",
-    "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency."
+    "glob": "glob <12 is deprecated and emits npm install warnings (https://github.com/isaacs/node-glob). Pulled in transitively via archiver-utils@5.0.2 (latest), which still pins glob@^10.0.0. archiver-utils only uses glob.sync(pattern, options), which remains compatible in glob@13. Remove this override once archiver-utils updates its glob dependency.",
+    "fast-xml-parser": "GHSA-8gc5-j5rx-235r, GHSA-jp2q-39xq-3w4g: fast-xml-parser <=5.5.6 has entity expansion bypass (CVE-2026-33036, CVE-2026-33349). Transitive via @aws-sdk/xml-builder. Remove once @aws-sdk updates to fast-xml-parser >=5.5.7.",
+    "@aws-sdk/xml-builder": "aws/aws-sdk-js-v3#7867: @aws-sdk/xml-builder <3.972.14 does not configure maxTotalExpansions on fast-xml-parser, causing 'Entity expansion limit exceeded' on large CloudFormation responses. Remove once @aws-sdk/client-* deps are bumped past 3.972.14."
   },
   "overrides": {
     "minimatch": "10.2.4",
-    "glob": "^13.0.0"
+    "glob": "^13.0.0",
+    "fast-xml-parser": "5.5.7",
+    "@aws-sdk/xml-builder": "3.972.15"
   },
   "engines": {
     "node": ">=20"

package.json

@@ -467,6 +467,7 @@ export interface GetEvaluatorResult {
     llmAsAJudge?: GetEvaluatorLlmConfig;
     codeBased?: GetEvaluatorCodeBasedConfig;
   };
+  kmsKeyArn?: string;
   tags?: Record<string, string>;
 }
 
@@ -545,6 +546,7 @@ export async function getEvaluator(options: GetEvaluatorOptions): Promise<GetEva
     status: response.status ?? 'UNKNOWN',
     description: response.description,
     evaluatorConfig,
+    kmsKeyArn: response.kmsKeyArn,
     tags,
   };
 }

src/cli/aws/agentcore-control.ts

@@ -210,6 +210,49 @@ describe('toEvaluatorSpec', () => {
 
     expect(result.tags).toBeUndefined();
   });
+
+  it('forwards kmsKeyArn when present', () => {
+    const detail: GetEvaluatorResult = {
+      evaluatorId: 'eval-kms',
+      evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-kms',
+      evaluatorName: 'kms_eval',
+      level: 'SESSION',
+      status: 'ACTIVE',
+      evaluatorConfig: {
+        llmAsAJudge: {
+          model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
+          instructions: 'Evaluate',
+          ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
+        },
+      },
+      kmsKeyArn: 'arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012',
+    };
+
+    const result = toEvaluatorSpec(detail, 'kms_eval');
+
+    expect(result.kmsKeyArn).toBe('arn:aws:kms:us-west-2:123456789012:key/12345678-1234-1234-1234-123456789012');
+  });
+
+  it('omits kmsKeyArn when not present', () => {
+    const detail: GetEvaluatorResult = {
+      evaluatorId: 'eval-no-kms',
+      evaluatorArn: 'arn:aws:bedrock-agentcore:us-west-2:123456789012:evaluator/eval-no-kms',
+      evaluatorName: 'no_kms_eval',
+      level: 'SESSION',
+      status: 'ACTIVE',
+      evaluatorConfig: {
+        llmAsAJudge: {
+          model: 'anthropic.claude-3-5-sonnet-20241022-v2:0',
+          instructions: 'Evaluate',
+          ratingScale: { numerical: [{ value: 1, label: 'Low', definition: 'Low' }] },
+        },
+      },
+    };
+
+    const result = toEvaluatorSpec(detail, 'no_kms_eval');
+
+    expect(result.kmsKeyArn).toBeUndefined();
+  });
 });
 
 // ============================================================================

src/cli/commands/import/__tests__/import-evaluator.test.ts

@@ -50,6 +50,7 @@ export function toEvaluatorSpec(detail: GetEvaluatorResult, localName: string):
     level,
     ...(detail.description && { description: detail.description }),
     config,
+    ...(detail.kmsKeyArn && { kmsKeyArn: detail.kmsKeyArn }),
     ...(detail.tags && Object.keys(detail.tags).length > 0 && { tags: detail.tags }),
   };
 }

src/cli/commands/import/import-evaluator.ts

@@ -1,6 +1,6 @@
 import { findConfigRoot } from '../../lib';
 import type { EvaluationLevel, Evaluator, EvaluatorConfig } from '../../schema';
-import { EvaluationLevelSchema, EvaluatorSchema } from '../../schema';
+import { EvaluationLevelSchema, EvaluatorSchema, isValidKmsKeyArn } from '../../schema';
 import { getErrorMessage } from '../errors';
 import type { RemovalPreview, RemovalResult, SchemaChange } from '../operations/remove/types';
 import { runCliCommand } from '../telemetry/cli-command-run.js';
@@ -25,6 +25,7 @@ export interface AddEvaluatorOptions {
   level: EvaluationLevel;
   description?: string;
   config: EvaluatorConfig;
+  kmsKeyArn?: string;
 }
 
 export type RemovableEvaluator = RemovableResource;
@@ -184,6 +185,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
         '--config <path>',
         'Path to evaluator config JSON file (overrides --model, --instructions, --rating-scale) [non-interactive]'
       )
+      .option('--kms-key-arn <arn>', 'KMS key ARN for evaluator encryption (optional)')
       .option('--json', 'Output as JSON [non-interactive]')
       .action(
         async (cliOptions: {
@@ -196,6 +198,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
           lambdaArn?: string;
           timeout?: string;
           config?: string;
+          kmsKeyArn?: string;
           json?: boolean;
         }) => {
           if (!findConfigRoot()) {
@@ -289,10 +292,17 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
                 };
               }
 
+              if (cliOptions.kmsKeyArn && !isValidKmsKeyArn(cliOptions.kmsKeyArn)) {
+                fail(
+                  '--kms-key-arn must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
+                );
+              }
+
               const result = await this.add({
                 name: cliOptions.name!,
                 level: levelResult.data!,
                 config: configJson,
+                kmsKeyArn: cliOptions.kmsKeyArn,
               });
 
               if (!result.success) {
@@ -386,6 +396,7 @@ export class EvaluatorPrimitive extends BasePrimitive<AddEvaluatorOptions, Remov
       level: options.level,
       ...(options.description && { description: options.description }),
       config: options.config,
+      ...(options.kmsKeyArn && { kmsKeyArn: options.kmsKeyArn }),
     };
 
     project.evaluators.push(evaluator);

src/cli/primitives/EvaluatorPrimitive.ts

@@ -8,6 +8,7 @@ interface CreateEvaluatorConfig {
   name: string;
   level: string;
   config: EvaluatorConfig;
+  kmsKeyArn?: string;
 }
 
 export function useCreateEvaluator() {
@@ -29,6 +30,7 @@ export function useCreateEvaluator() {
             name: config.name,
             level: config.level as 'SESSION' | 'TRACE' | 'TOOL_CALL',
             config: config.config,
+            kmsKeyArn: config.kmsKeyArn,
           })
       );
       if (!addResult.success) {

src/cli/tui/hooks/useCreateEvaluator.ts

@@ -1,5 +1,5 @@
 import type { EvaluationLevel, EvaluatorConfig } from '../../../../schema';
-import { EvaluatorNameSchema, isValidBedrockModelId } from '../../../../schema';
+import { EvaluatorNameSchema, isValidBedrockModelId, isValidKmsKeyArn } from '../../../../schema';
 import type { SelectableItem } from '../../components';
 import { ConfirmReview, Panel, Screen, StepIndicator, TextInput, WizardSelect } from '../../components';
 import { HELP_TEXT } from '../../constants';
@@ -91,6 +91,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
   const isRatingScaleCustomStep = wizard.step === 'ratingScale-custom';
   const isLambdaArnStep = wizard.step === 'lambda-arn';
   const isTimeoutStep = wizard.step === 'timeout';
+  const isKmsKeyArnStep = wizard.step === 'kms-key-arn';
   const isConfirmStep = wizard.step === 'confirm';
 
   const evaluatorTypeNav = useListNavigation({
@@ -163,6 +164,8 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
 
   // Build confirm fields based on evaluator type
   const confirmFields = useMemo(() => {
+    const kmsField = wizard.config.kmsKeyArn ? [{ label: 'KMS Key ARN', value: wizard.config.kmsKeyArn }] : [];
+
     if (wizard.evaluatorType === 'llm-as-a-judge') {
       const llm = wizard.config.config.llmAsAJudge!;
       return [
@@ -175,6 +178,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
           value: llm.instructions.length > 60 ? llm.instructions.slice(0, 60) + '...' : llm.instructions,
         },
         { label: 'Rating Scale', value: formatRatingScale(llm.ratingScale) },
+        ...kmsField,
       ];
     }
 
@@ -187,6 +191,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
         { label: 'Code', value: managed.codeLocation },
         { label: 'Entrypoint', value: managed.entrypoint },
         { label: 'Timeout', value: `${managed.timeoutSeconds}s` },
+        ...kmsField,
       ];
     }
 
@@ -197,6 +202,7 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
       { label: 'Name', value: wizard.config.name },
       { label: 'Level', value: wizard.config.level },
       { label: 'Lambda ARN', value: external.lambdaArn },
+      ...kmsField,
     ];
   }, [wizard.evaluatorType, wizard.codeBasedType, wizard.config]);
 
@@ -374,6 +380,21 @@ export function AddEvaluatorScreen({ onComplete, onExit, existingEvaluatorNames
           />
         )}
 
+        {isKmsKeyArnStep && (
+          <TextInput
+            key="kms-key-arn"
+            prompt="KMS key ARN for encryption (optional, press Enter to skip)"
+            initialValue=""
+            onSubmit={wizard.setKmsKeyArn}
+            onCancel={() => wizard.goBack()}
+            customValidation={value =>
+              value === '' ||
+              isValidKmsKeyArn(value) ||
+              'Must be a valid KMS key ARN (e.g. arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012)'
+            }
+          />
+        )}
+
         {isConfirmStep && <ConfirmReview fields={confirmFields} />}
       </Panel>
     </Screen>

src/cli/tui/screens/evaluator/AddEvaluatorScreen.tsx

@@ -20,12 +20,14 @@ export type AddEvaluatorStep =
   | 'ratingScale-custom'
   | 'lambda-arn'
   | 'timeout'
+  | 'kms-key-arn'
   | 'confirm';
 
 export interface AddEvaluatorConfig {
   name: string;
   level: EvaluationLevel;
   config: EvaluatorConfig;
+  kmsKeyArn?: string;
 }
 
 export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
@@ -41,6 +43,7 @@ export const EVALUATOR_STEP_LABELS: Record<AddEvaluatorStep, string> = {
   'ratingScale-custom': 'Scale',
   'lambda-arn': 'Lambda',
   timeout: 'Timeout',
+  'kms-key-arn': 'KMS Key',
   confirm: 'Confirm',
 };
 

src/cli/tui/screens/evaluator/types.ts

@@ -22,6 +22,7 @@ const LLM_STEPS: AddEvaluatorStep[] = [
   'model',
   'instructions',
   'ratingScale',
+  'kms-key-arn',
   'confirm',
 ];
 const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
@@ -30,6 +31,7 @@ const CODE_MANAGED_STEPS: AddEvaluatorStep[] = [
   'name',
   'level',
   'timeout',
+  'kms-key-arn',
   'confirm',
 ];
 const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
@@ -38,6 +40,7 @@ const CODE_EXTERNAL_STEPS: AddEvaluatorStep[] = [
   'name',
   'level',
   'lambda-arn',
+  'kms-key-arn',
   'confirm',
 ];
 
@@ -80,6 +83,7 @@ export function useAddEvaluatorWizard() {
   const [lambdaArn, setLambdaArnState] = useState('');
   const [timeout, setTimeoutState] = useState(DEFAULT_CODE_TIMEOUT);
   const [customRatingScaleType, setCustomRatingScaleType] = useState<CustomRatingScaleType>('numerical');
+  const [kmsKeyArn, setKmsKeyArnState] = useState('');
   const [step, setStep] = useState<AddEvaluatorStep>('evaluator-type');
 
   const steps = useMemo(() => getSteps(evaluatorType, codeBasedType), [evaluatorType, codeBasedType]);
@@ -109,11 +113,13 @@ export function useAddEvaluatorWizard() {
 
   // Build the final config based on current state
   const config: AddEvaluatorConfig = useMemo(() => {
+    const kms = kmsKeyArn || undefined;
     if (evaluatorType === 'llm-as-a-judge') {
       return {
         name,
         level,
         config: { llmAsAJudge: llmConfig },
+        ...(kms && { kmsKeyArn: kms }),
       };
     }
 
@@ -126,6 +132,7 @@ export function useAddEvaluatorWizard() {
             external: { lambdaArn },
           },
         },
+        ...(kms && { kmsKeyArn: kms }),
       };
     }
 
@@ -143,8 +150,9 @@ export function useAddEvaluatorWizard() {
           },
         },
       },
+      ...(kms && { kmsKeyArn: kms }),
     };
-  }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout]);
+  }, [evaluatorType, codeBasedType, name, level, llmConfig, lambdaArn, timeout, kmsKeyArn]);
 
   const selectEvaluatorType = useCallback((type: EvaluatorTypeId) => {
     setEvaluatorType(type);
@@ -256,6 +264,15 @@ export function useAddEvaluatorWizard() {
     [nextStep]
   );
 
+  const setKmsKeyArn = useCallback(
+    (arn: string) => {
+      setKmsKeyArnState(arn);
+      const next = nextStep('kms-key-arn');
+      if (next) setStep(next);
+    },
+    [nextStep]
+  );
+
   const reset = useCallback(() => {
     setEvaluatorType('code-based');
     setCodeBasedType('managed');
@@ -264,6 +281,7 @@ export function useAddEvaluatorWizard() {
     setLlmConfig(getDefaultLlmConfig().llmAsAJudge!);
     setLambdaArnState('');
     setTimeoutState(DEFAULT_CODE_TIMEOUT);
+    setKmsKeyArnState('');
     setStep('evaluator-type');
   }, []);
 
@@ -288,6 +306,7 @@ export function useAddEvaluatorWizard() {
     setCustomRatingScale,
     setLambdaArn,
     setTimeout,
+    setKmsKeyArn,
     reset,
   };
 }