fix: allow code-based evaluators in online eval configs (#947)

* fix: allow code-based evaluators in online eval configs

Remove restrictions that blocked code-based evaluators from being used
in online evaluation configs. The service now supports code-based
evaluators for online evaluation.

Changes:
- Remove code-based evaluator block in OnlineEvalConfigPrimitive
- Remove code-based evaluator validation in schema superRefine
- Remove code-based evaluator filter in TUI evaluator picker

* style: fix prettier formatting

Author: jariy17

src/cli/primitives/OnlineEvalConfigPrimitive.ts

@@ -211,18 +211,6 @@ export class OnlineEvalConfigPrimitive extends BasePrimitive<AddOnlineEvalConfig
 
     this.checkDuplicate(project.onlineEvalConfigs, options.name, 'Online eval config');
 
-    // Block code-based evaluators — only LLM-as-a-Judge evaluators are supported for online evaluation.
-    // Checks local project config. ARN-based evaluators are filtered in the TUI by API evaluatorType.
-    // TODO: For ARN-based evaluators in non-interactive mode, call getEvaluator to check type.
-    for (const evalName of options.evaluators) {
-      const evaluator = project.evaluators.find(e => e.name === evalName);
-      if (evaluator?.config.codeBased) {
-        throw new Error(
-          `Code-based evaluator "${evalName}" cannot be used in online eval configs. Only LLM-as-a-Judge evaluators are supported for online evaluation.`
-        );
-      }
-    }
-
     const config: OnlineEvalConfig = {
       name: options.name,
       agent: options.agent,

src/cli/tui/screens/online-eval/AddOnlineEvalFlow.tsx

@@ -48,17 +48,12 @@ export function AddOnlineEvalFlow({ isInteractive = true, onExit, onBack, onDev,
         const result = await listEvaluators({ region });
         if (cancelled) return;
 
-        // Filter out code-based evaluators — not supported for online evaluation.
-        // Check both the API response type ('CustomCode') and local config (codeBased).
-        const codeBasedNames = new Set(projectSpec.evaluators.filter(e => e.config.codeBased).map(e => e.name));
-        const items: EvaluatorItem[] = result.evaluators
-          .filter(e => e.evaluatorType !== 'CustomCode' && !codeBasedNames.has(e.evaluatorName))
-          .map(e => ({
-            arn: e.evaluatorArn,
-            name: e.evaluatorName,
-            type: e.evaluatorType,
-            description: e.description,
-          }));
+        const items: EvaluatorItem[] = result.evaluators.map(e => ({
+          arn: e.evaluatorArn,
+          name: e.evaluatorName,
+          type: e.evaluatorType,
+          description: e.description,
+        }));
 
         const agentNames = projectSpec.runtimes.map(a => a.name);
 

src/schema/schemas/agentcore-project.ts

@@ -337,15 +337,6 @@ export const AgentCoreProjectSpecSchema = z
             message: `Online eval config "${config.name}" references unknown evaluator "${evalName}"`,
           });
         }
-
-        // Block code-based evaluators in online eval configs
-        const evaluator = spec.evaluators.find(e => e.name === evalName);
-        if (evaluator?.config.codeBased) {
-          ctx.addIssue({
-            code: z.ZodIssueCode.custom,
-            message: `Online eval config "${config.name}" references code-based evaluator "${evalName}". Code-based evaluators are not supported for online evaluation.`,
-          });
-        }
       }
     }
   });