fix(sdk): align eval config providers surface (#1722)

* fix(sdk): align eval config providers surface

* fix(sdk): use inline TS providers during evaluate

Author: christso

apps/web/src/content/docs/docs/next/evaluation/sdk.mdx

@@ -97,13 +97,28 @@ import { graders, type EvalConfig } from '@agentv/sdk';
 
 const config: EvalConfig = {
   name: 'hello-suite',
-  providers: ['mock-sdk'],
+  providers: [
+    { id: 'mock', label: 'mock-sdk', config: { response: 'Hello from the mock provider' } },
+    { id: 'openai:gpt-5-mini', label: 'grader-provider' },
+  ],
+  defaults: {
+    provider: 'mock-sdk',
+    grader: 'grader-provider',
+  },
+  defaultTest: {
+    options: {
+      provider: 'grader-provider',
+    },
+  },
   prompts: ['{{ task }}'],
   tests: [
     {
       id: 'hello',
       vars: { task: 'Say hello' },
       inputFiles: ['../fixtures/per-test-note.md'],
+      options: {
+        provider: 'grader-provider',
+      },
       assert: [graders.contains('Hello')],
     },
   ],
@@ -120,6 +135,8 @@ Useful companion helpers:
 
 The durable authored field remains `assert`. TypeScript eval config authoring does not introduce a second YAML vocabulary.
 
+TypeScript eval configs use the same provider surface as YAML: top-level `providers` defines both systems under test and reusable grader providers, `providers[].id` names the backend/spec, `providers[].label` is the stable AgentV identity, and `defaults.provider` / `defaults.grader` select the default candidate and grader. Per-test grader provider selection belongs in `defaultTest.options.provider`, `tests[].options.provider`, or assertion-level `provider`.
+
 ## Built-In Grader Helpers
 
 `@agentv/sdk` includes a small `graders` catalog for common deterministic and LLM-backed grader configs. These helpers return ordinary `assert` entries and serialize to the same canonical YAML you could write by hand.

packages/core/src/evaluation/evaluate.ts

@@ -222,6 +222,7 @@ export interface MaterializedEvalConfig {
   readonly threshold?: number;
   readonly metadata?: EvalMetadata;
   readonly target?: ProviderDefinition;
+  readonly targets?: readonly ProviderDefinition[];
   readonly task?: (input: string) => string | Promise<string>;
   readonly providerFactory?: ProviderFactoryFn;
 }
@@ -362,6 +363,7 @@ export async function evaluate(config: EvalConfig): Promise<EvalRunResult> {
     testFilePath,
     repoRoot,
     target: resolvedTarget,
+    ...(materialized.targets ? { targets: materialized.targets } : {}),
     ...(providerFactory ? { providerFactory } : {}),
     maxRetries: config.maxRetries ?? 2,
     agentTimeoutMs: config.agentTimeoutMs,
@@ -426,6 +428,9 @@ export async function materializeEvalConfig(
       category: options?.category,
     });
     const tests = applyProgrammaticSuiteOverrides(suite.tests, config);
+    const suiteTargetDefinitions = suite.targetRefs
+      ?.map((targetRef) => targetRef.definition)
+      .filter((definition): definition is ProviderDefinition => definition !== undefined);
     return {
       testFilePath,
       tests,
@@ -435,7 +440,10 @@ export async function materializeEvalConfig(
       budgetUsd: config.budgetUsd ?? suite.budgetUsd,
       threshold: config.threshold ?? suite.threshold,
       metadata: config.metadata ?? suite.metadata,
-      target: config.target ?? suite.inlineTarget,
+      target: config.target ?? suite.inlineTarget ?? suiteTargetDefinitions?.[0],
+      ...(suiteTargetDefinitions && suiteTargetDefinitions.length > 0
+        ? { targets: suiteTargetDefinitions }
+        : {}),
       task: config.task,
       providerFactory: suite.providerFactory,
     };

packages/core/src/evaluation/loaders/grader-parser.ts

@@ -238,6 +238,7 @@ export async function parseGraders(
 ): Promise<readonly GraderConfig[] | undefined> {
   const execution = rawEvalCase.execution;
   const executionObject = isJsonObject(execution) ? execution : undefined;
+  const inheritedAssertionConfig = inheritedAssertionConfigFromOptions(rawEvalCase.options);
 
   // Case-level graders priority: assert > execution assert.
   const caseEvaluators =
@@ -255,7 +256,7 @@ export async function parseGraders(
     evalId,
     defaultPreprocessors,
     defaultRubricPrompt,
-    undefined,
+    inheritedAssertionConfig,
     inheritedGraderTarget,
   );
   // Parse root-level evaluators (appended after case-level)
@@ -265,7 +266,7 @@ export async function parseGraders(
     evalId,
     defaultPreprocessors,
     defaultRubricPrompt,
-    undefined,
+    inheritedAssertionConfig,
     inheritedGraderTarget,
   );
 
@@ -1872,21 +1873,45 @@ function withInheritedAssertionConfig(
   inheritedConfig?: JsonObject,
 ): JsonObject {
   const ownConfig = isJsonObject(rawEvaluator.config) ? rawEvaluator.config : undefined;
-  if (!inheritedConfig && !ownConfig) {
+  const inheritedProvider =
+    typeof inheritedConfig?.provider === 'string' && inheritedConfig.provider.trim().length > 0
+      ? inheritedConfig.provider.trim()
+      : undefined;
+  const inheritedConfigWithoutProvider = inheritedConfig
+    ? Object.fromEntries(Object.entries(inheritedConfig).filter(([key]) => key !== 'provider'))
+    : undefined;
+  const inheritedConfigForConfig =
+    inheritedConfigWithoutProvider && Object.keys(inheritedConfigWithoutProvider).length > 0
+      ? inheritedConfigWithoutProvider
+      : undefined;
+  if (!inheritedConfigForConfig && !ownConfig && inheritedProvider === undefined) {
     return rawEvaluator;
   }
 
   const mergedConfig = {
-    ...(inheritedConfig ?? {}),
+    ...(inheritedConfigForConfig ?? {}),
     ...(ownConfig ?? {}),
   };
 
   return {
     ...rawEvaluator,
-    config: mergedConfig,
+    ...(rawEvaluator.provider === undefined && inheritedProvider !== undefined
+      ? { provider: inheritedProvider }
+      : {}),
+    ...(Object.keys(mergedConfig).length > 0 ? { config: mergedConfig } : {}),
   };
 }
 
+function inheritedAssertionConfigFromOptions(
+  options: JsonValue | undefined,
+): JsonObject | undefined {
+  if (!isJsonObject(options)) {
+    return undefined;
+  }
+  const provider = typeof options.provider === 'string' ? options.provider.trim() : '';
+  return provider.length > 0 ? { provider } : undefined;
+}
+
 interface ParsedPromptField {
   readonly prompt?: string;
   readonly promptPath?: string;

packages/core/src/evaluation/loaders/ts-eval-loader.ts

@@ -30,6 +30,7 @@ const KNOWN_SNAKE_CASE_KEYS = {
   budgetUsd: 'budget_usd',
   conversationId: 'conversation_id',
   costLimitUsd: 'cost_limit_usd',
+  defaultTest: 'default_test',
   dependsOn: 'depends_on',
   earlyExit: 'early_exit',
   expectedOutput: 'expected_output',
@@ -45,6 +46,7 @@ const KNOWN_SNAKE_CASE_KEYS = {
   outputPath: 'output_path',
   readOnly: 'read_only',
   reasoningEffort: 'reasoning_effort',
+  rubricPrompt: 'rubric_prompt',
   skipDefaults: 'skip_defaults',
   timeoutMs: 'timeout_ms',
   timeoutSeconds: 'timeout_seconds',
@@ -193,10 +195,7 @@ function isProgrammaticEvalConfig(value: unknown): value is ProgrammaticEvalConf
 
 function lowerTypeScriptEvalConfig(config: Record<string, unknown>): Record<string, unknown> {
   const lowered = lowerEvalYamlValue(config) as Record<string, unknown>;
-  const { budget_usd: budgetUsd, repeat, target, ...withoutRuntimeAliases } = lowered;
-  if (target !== undefined && withoutRuntimeAliases.providers === undefined) {
-    withoutRuntimeAliases.providers = [target];
-  }
+  const { budget_usd: budgetUsd, repeat, ...withoutRuntimeAliases } = lowered;
   if (budgetUsd === undefined && repeat === undefined) {
     return withoutRuntimeAliases;
   }

packages/core/src/evaluation/yaml-parser.ts

@@ -2653,7 +2653,12 @@ function readSuiteRuntimeBlock(suite: RawTestSuite, evalFilePath: string): JsonO
   }
   if (suite.model !== undefined) {
     throw new Error(
-      `Invalid eval runtime config in ${evalFilePath}: top-level 'model' is not part of eval YAML. Put model inside the target object.`,
+      `Invalid eval runtime config in ${evalFilePath}: top-level 'model' is not part of eval YAML. Put model inside the relevant providers[].config object.`,
+    );
+  }
+  if ((suite as Record<string, unknown>).graders !== undefined) {
+    throw new Error(
+      `Invalid eval runtime config in ${evalFilePath}: top-level 'graders' has been removed. Put grader providers in 'providers' and select them with defaults.grader, default_test.options.provider, tests[].options.provider, or assertion provider.`,
     );
   }
   if (suite.runs !== undefined) {

packages/core/test/evaluation/loaders/fixtures/default-export.eval.ts

@@ -1,23 +1,23 @@
-import type { EvalConfig } from '../../../../src/evaluation/evaluate.js';
-
-const config: EvalConfig = {
-  metadata: {
-    name: 'default-export-suite',
-    tags: ['sdk', 'typescript'],
-  },
+const config = {
+  name: 'default-export-suite',
+  tags: ['sdk', 'typescript'],
   prompts: ['{{ input }}'],
+  providers: [
+    {
+      id: 'mock',
+      label: 'inline-provider',
+      config: { response: 'hello there' },
+    },
+  ],
   tests: [
     {
       id: 'greeting',
       vars: { input: 'Say hello' },
       assert: [{ type: 'contains', value: 'hello' }],
     },
   ],
-  cache: false,
-  cachePath: '.agentv/ts-eval-cache',
   budgetUsd: 1.5,
   threshold: 0.9,
-  target: { name: 'inline-target', provider: 'mock', response: 'hello there' },
 };
 
 export default config;

packages/core/test/evaluation/loaders/fixtures/eval-config-named.eval.ts

@@ -1,13 +1,11 @@
-import type { EvalConfig } from '../../../../src/evaluation/evaluate.js';
-
-export const evalConfig: EvalConfig = {
+export const evalConfig = {
   prompts: ['{{ input }}'],
+  providers: ['mock-provider'],
   tests: [
     {
       id: 'eval-config-named',
       vars: { input: 'Say hello' },
       assert: [{ type: 'contains', value: 'hello' }],
     },
   ],
-  target: { provider: 'mock_agent' },
 };

packages/core/test/evaluation/loaders/fixtures/legacy-graders.eval.ts

@@ -0,0 +1,13 @@
+export default {
+  name: 'legacy-graders',
+  providers: ['mock-provider'],
+  graders: [{ id: 'mock', label: 'grader-provider' }],
+  prompts: ['{{ input }}'],
+  tests: [
+    {
+      id: 'legacy-graders',
+      vars: { input: 'Say hello' },
+      assert: [{ type: 'contains', value: 'hello' }],
+    },
+  ],
+};

packages/core/test/evaluation/loaders/fixtures/legacy-target.eval.ts

@@ -0,0 +1,12 @@
+export default {
+  name: 'legacy-target',
+  target: 'mock-target',
+  prompts: ['{{ input }}'],
+  tests: [
+    {
+      id: 'legacy-target',
+      vars: { input: 'Say hello' },
+      assert: [{ type: 'contains', value: 'hello' }],
+    },
+  ],
+};

packages/core/test/evaluation/loaders/fixtures/legacy-targets.eval.ts

@@ -0,0 +1,12 @@
+export default {
+  name: 'legacy-targets',
+  targets: ['mock-target'],
+  prompts: ['{{ input }}'],
+  tests: [
+    {
+      id: 'legacy-targets',
+      vars: { input: 'Say hello' },
+      assert: [{ type: 'contains', value: 'hello' }],
+    },
+  ],
+};