feat(core): support promptfoo provider declarations

Accept Promptfoo-compatible provider strings and maps while preserving AgentV-only export diagnostics.

Author: christso

apps/web/src/content/docs/docs/next/targets/configuration.mdx

@@ -295,20 +295,50 @@ tests:
   - id: test-2
 ```
 
-The string is a configured provider `label` or `id`. Use object form when an
-eval needs a local provider variant:
+The string is usually a configured provider `label` or `id`. Promptfoo-style
+provider spec strings such as `openai:gpt-4.1-mini` are also accepted and lower
+to an inline provider definition. Use object form when an eval needs a local
+provider variant:
 
 ```yaml
 providers:
   - id: codex-app-server
     label: codex-high-reasoning
-  runtime: host
-  config:
-    command: ["codex", "app-server"]
-    model: gpt-5-codex
-    reasoning_effort: high
+    runtime: host
+    config:
+      command: ["codex", "app-server"]
+      model: gpt-5-codex
+      reasoning_effort: high
 ```
 
+AgentV accepts the Promptfoo provider declaration layer where the semantics
+match: strings, object form, provider maps, and `inputs`. In object form, `id`
+is the backend/spec and `label` is the stable AgentV result and selection
+identity.
+
+```yaml
+providers:
+  - openai:gpt-4.1-mini
+  - openai:gpt-4:
+      label: gpt4-low-temp
+      config:
+        temperature: 0
+      inputs:
+        prompt: User prompt text
+  - id: openai:responses:gpt-5.4
+    label: gpt5-responses
+    inputs:
+      prompt:
+        type: text
+        description: User prompt text
+```
+
+`runtime`, provider-local `environment`, and provider `hooks` are AgentV
+extensions. Direct Promptfoo runs do not execute those fields. A Promptfoo
+export path must either lower a supported AgentV extension into Promptfoo
+providers/extensions or reject the config with a clear diagnostic instead of
+silently dropping runtime or testbed behavior.
+
 Use `defaults.grader` for the project default grader. `default_test.options.provider`
 and `tests[].options.provider` can choose a grader provider for LLM-backed
 assertions before an assertion-level `provider` override takes final precedence.
@@ -443,7 +473,6 @@ Provider hooks can be scoped to an eval-local provider object:
 providers:
   - id: codex-app-server
     label: codex-with-skills
-    extends: default
     hooks:
       before_each:
         command: ["setup-plugins.sh", "skills"]

packages/core/src/evaluation/loaders/config-graph.ts

@@ -2,7 +2,7 @@ import { readFile } from 'node:fs/promises';
 import path from 'node:path';
 
 import { isPlainConfigObject } from '../../config-overlays.js';
-import { normalizeProviderDefinition } from '../providers/targets.js';
+import { expandProviderDefinitionEntries } from '../providers/targets.js';
 import { parseYamlValue } from '../yaml-loader.js';
 
 const FILE_PROTOCOL = 'file://';
@@ -214,12 +214,17 @@ function parseArray(value: unknown, location: string): readonly unknown[] {
 }
 
 function parseProviders(value: unknown, location: string): readonly NormalizedTargetConfig[] {
-  return parseArray(value, location).map((entry, index) =>
-    parseProvider(entry, `${location}[${index}]`),
-  );
+  return expandProviderDefinitionEntries(parseArray(value, location), {
+    location,
+    stringMode: 'all',
+  }).map((entry) => parseProvider(entry.rawDefinition, entry.definition, `${location}`));
 }
 
-function parseProvider(value: unknown, location: string): NormalizedTargetConfig {
+function parseProvider(
+  value: unknown,
+  definition: ReturnType<typeof expandProviderDefinitionEntries>[number]['definition'],
+  location: string,
+): NormalizedTargetConfig {
   if (!isPlainConfigObject(value)) {
     throw new Error(`Invalid ${location}: provider must be an object.`);
   }
@@ -229,7 +234,6 @@ function parseProvider(value: unknown, location: string): NormalizedTargetConfig
     }
   }
 
-  const definition = normalizeProviderDefinition(value, { location });
   const id = definition.name;
   const provider = readRequiredString(definition.provider, `${location}.id`);
   if (AMBIGUOUS_PROVIDER_ALIASES.has(provider)) {

packages/core/src/evaluation/loaders/config-loader.ts

@@ -12,7 +12,8 @@ import {
 import { getAgentvConfigDir } from '../../paths.js';
 import { createEvalConfigEnv, interpolateEnv } from '../interpolation.js';
 import {
-  normalizeProviderDefinition,
+  expandProviderDefinitionEntries,
+  isProviderSpecString,
   resolveProviderDefinitionEnvironments,
 } from '../providers/targets.js';
 import type { ProviderDefinition } from '../providers/types.js';
@@ -326,14 +327,44 @@ function parseProviderDefinitions(
   if (!Array.isArray(rawProviders)) {
     return Promise.resolve(undefined);
   }
-  const definitions = rawProviders.map((entry, index) =>
-    normalizeProviderDefinition(entry, { location: `${configPath}:providers[${index}]` }),
-  );
+  const definitions = expandProviderDefinitionEntries(rawProviders, {
+    location: `${configPath}:providers`,
+    stringMode: 'all',
+  }).map((entry) => entry.definition);
   return resolveProviderDefinitionEnvironments(definitions, baseDir, {
     location: `${configPath}:providers`,
   });
 }
 
+function parseInlineProviderRefs(rawProviders: readonly unknown[]): readonly EvalTargetRef[] {
+  const refs: EvalTargetRef[] = [];
+  rawProviders.forEach((entry, index) => {
+    const location = `providers[${index}]`;
+    if (typeof entry === 'string' && !isProviderSpecString(entry)) {
+      const name = entry.trim();
+      if (name.length === 0) {
+        throw new Error(`Invalid ${location}: provider reference must be non-empty.`);
+      }
+      refs.push({ name });
+      return;
+    }
+
+    refs.push(...parseEvalProviderRefs(entry, location));
+  });
+  return refs;
+}
+
+function parseEvalProviderRefs(raw: unknown, location: string): readonly EvalTargetRef[] {
+  const entries = expandProviderDefinitionEntries([raw], {
+    location: location.replace(/\[\d+\]$/, ''),
+    stringMode: 'spec-only',
+  });
+
+  return entries.map((entry) =>
+    providerDefinitionToRef(entry.rawDefinition, entry.rawId, entry.definition),
+  );
+}
+
 function mergeExecutionConfig(
   defaults: ExecutionDefaults | undefined,
   graph: ComposableConfigGraph['execution'],
@@ -543,7 +574,7 @@ export function extractTargetRefsFromSuite(
   }
 
   const entries = Array.isArray(rawProviders) ? rawProviders : [rawProviders];
-  const refs = entries.map((entry, index) => parseEvalProviderRef(entry, `providers[${index}]`));
+  const refs = parseInlineProviderRefs(entries);
   assertUniqueProviderRefs(refs);
   return refs.length > 0 ? refs : undefined;
 }
@@ -558,29 +589,18 @@ export function extractTargetsFromSuite(suite: JsonObject): readonly string[] |
   return names.length > 0 ? names : undefined;
 }
 
-function parseEvalProviderRef(raw: unknown, location: string): EvalTargetRef {
-  if (typeof raw === 'string') {
-    const name = raw.trim();
-    if (name.length === 0) {
-      throw new Error(`Invalid ${location}: provider reference must be non-empty.`);
-    }
-    return { name };
-  }
-
-  if (!isJsonObject(raw)) {
-    throw new Error(`Invalid ${location}: use a provider reference string or provider object.`);
-  }
-
+function providerDefinitionToRef(
+  raw: Record<string, unknown>,
+  rawId: string,
+  definition: ProviderDefinition,
+): EvalTargetRef {
   const hooks = parseTargetHooks(raw.hooks);
-  const definition = normalizeProviderDefinition(
-    Object.fromEntries(Object.entries(raw).filter(([key]) => key !== 'hooks')),
-    { location },
-  ) as ProviderDefinition;
-
+  const rawLabel =
+    typeof raw.label === 'string' && raw.label.trim().length > 0 ? raw.label.trim() : undefined;
   return {
     name: definition.name,
-    id: typeof raw.id === 'string' ? raw.id.trim() : definition.provider,
-    ...(definition.label !== undefined ? { label: definition.label } : {}),
+    id: rawId,
+    ...(rawLabel !== undefined ? { label: rawLabel } : {}),
     definition,
     ...(hooks !== undefined ? { hooks } : {}),
   };

packages/core/src/evaluation/providers/targets-file.ts

@@ -3,7 +3,10 @@ import { access, readFile } from 'node:fs/promises';
 import path from 'node:path';
 
 import { parseYamlValue } from '../yaml-loader.js';
-import { normalizeProviderDefinition, resolveProviderDefinitionEnvironments } from './targets.js';
+import {
+  expandProviderDefinitionEntries,
+  resolveProviderDefinitionEnvironments,
+} from './targets.js';
 import { TARGETS_SCHEMA_V2 } from './types.js';
 import type { ProviderDefinition } from './types.js';
 
@@ -32,24 +35,6 @@ function extractProvidersArray(parsed: unknown, absolutePath: string): unknown[]
   return providers;
 }
 
-function assertProviderDefinition(
-  value: unknown,
-  index: number,
-  filePath: string,
-): ProviderDefinition {
-  if (!isRecord(value)) {
-    throw new Error(`providers entry at index ${index} in ${filePath} must be an object`);
-  }
-
-  const id = value.id;
-
-  if (typeof id !== 'string' || id.trim().length === 0) {
-    throw new Error(`providers entry at index ${index} in ${filePath} is missing a valid 'id'`);
-  }
-
-  return normalizeProviderDefinition(value, { location: `providers[${index}]` });
-}
-
 async function fileExists(filePath: string): Promise<boolean> {
   try {
     await access(filePath, constants.F_OK);
@@ -71,9 +56,10 @@ export async function readProviderDefinitions(
   const parsed = parseYamlValue(raw);
 
   const providers = extractProvidersArray(parsed, absolutePath);
-  const definitions = providers.map((entry, index) =>
-    assertProviderDefinition(entry, index, absolutePath),
-  );
+  const definitions = expandProviderDefinitionEntries(providers, {
+    location: 'providers',
+    stringMode: 'all',
+  }).map((entry) => entry.definition);
   return resolveProviderDefinitionEnvironments(definitions, path.dirname(absolutePath), {
     location: 'providers',
   });

packages/core/src/evaluation/providers/targets.ts

@@ -699,6 +699,99 @@ export interface NormalizeProviderDefinitionOptions {
   readonly location?: string;
 }
 
+export type ExpandedProviderDefinitionEntry = {
+  readonly rawId: string;
+  readonly rawDefinition: Record<string, unknown>;
+  readonly definition: ProviderDefinition;
+};
+
+export function isProviderSpecString(value: string): boolean {
+  const trimmed = value.trim();
+  return trimmed.startsWith('package:') || trimmed.startsWith('file://') || trimmed.includes(':');
+}
+
+export function expandProviderDefinitionEntries(
+  entries: readonly unknown[],
+  options: {
+    readonly location?: string;
+    readonly stringMode?: 'all' | 'spec-only';
+  } = {},
+): readonly ExpandedProviderDefinitionEntry[] {
+  const location = options.location ?? 'providers';
+  const stringMode = options.stringMode ?? 'all';
+  const expanded: ExpandedProviderDefinitionEntry[] = [];
+
+  entries.forEach((entry, index) => {
+    const entryLocation = `${location}[${index}]`;
+    if (typeof entry === 'string') {
+      const id = entry.trim();
+      if (id.length === 0) {
+        throw new Error(`Invalid ${entryLocation}: provider string must be non-empty.`);
+      }
+      if (stringMode === 'spec-only' && !isProviderSpecString(id)) {
+        return;
+      }
+      const rawDefinition = { id };
+      expanded.push({
+        rawId: id,
+        rawDefinition,
+        definition: normalizeProviderDefinition(rawDefinition, { location: entryLocation }),
+      });
+      return;
+    }
+
+    if (!isRecord(entry)) {
+      throw new Error(`Invalid ${entryLocation}: provider must be a string or object.`);
+    }
+
+    if (typeof entry.id === 'string' && entry.id.trim().length > 0) {
+      expanded.push({
+        rawId: entry.id.trim(),
+        rawDefinition: entry,
+        definition: normalizeProviderDefinition(entry, { location: entryLocation }),
+      });
+      return;
+    }
+
+    if (
+      entry.provider !== undefined ||
+      entry.name !== undefined ||
+      entry.container !== undefined ||
+      entry.install !== undefined
+    ) {
+      normalizeProviderDefinition(entry, { location: entryLocation });
+      return;
+    }
+
+    const mapEntries = Object.entries(entry);
+    if (mapEntries.length === 0) {
+      throw new Error(`Invalid ${entryLocation}: provider map must not be empty.`);
+    }
+
+    for (const [providerId, providerOptions] of mapEntries) {
+      if (providerId.trim().length === 0) {
+        throw new Error(`Invalid ${entryLocation}: provider map key must be non-empty.`);
+      }
+      if (!isRecord(providerOptions)) {
+        throw new Error(
+          `Invalid ${entryLocation}.${providerId}: provider map value must be an object.`,
+        );
+      }
+      const { id: _ignoredId, ...optionsWithoutId } = providerOptions;
+      const rawDefinition = { ...optionsWithoutId, id: providerId };
+      expanded.push({
+        rawId: providerId,
+        rawDefinition,
+        definition: normalizeProviderDefinition(rawDefinition, {
+          location: `${entryLocation}.${providerId}`,
+        }),
+      });
+    }
+  });
+
+  return expanded;
+}
+
 function normalizePublicProviderId(providerId: string): {
   readonly provider: string;
   readonly config: Record<string, unknown>;

packages/core/src/evaluation/providers/types.ts

@@ -466,6 +466,7 @@ export interface ProviderDefinition {
   readonly prompts?: unknown | undefined;
   readonly transform?: unknown | undefined;
   readonly delay?: number | unknown | undefined;
+  readonly inputs?: unknown | undefined;
   readonly provider?: ProviderKind | string;
   /** Original public providers[].id backend/spec string after public provider normalization. */
   readonly provider_spec?: string | undefined;