From 76bdb9a14d136641f6da3586c7a8ce08baf7c33e Mon Sep 17 00:00:00 2001 From: Christopher Date: Mon, 6 Jul 2026 14:32:27 +1000 Subject: [PATCH] feat(config): add v5-native env_path/env_from environment injection Adds explicit .agentv/config.yaml support for env_path (dotenv file loading) and env_from (argv command output injection), replacing the ad hoc hooks.before_session shell hook as the supported way to feed secrets into {{ env.* }} interpolation before validate/eval. - env_path loads dotenv-style files; missing files warn, don't fail. - env_from runs argv commands (shell strings rejected) and parses shell_exports or json stdout into process.env. - Existing process.env always wins; injected values are never logged. - hooks.before_session keeps running, now with a deprecation warning, and config validation no longer flags hooks as an unexpected field. Co-Authored-By: Claude Sonnet 5 --- apps/cli/src/index.ts | 21 +- .../docs/docs/next/targets/configuration.mdx | 47 +++ packages/core/src/evaluation/env-injection.ts | 214 +++++++++++++ .../src/evaluation/loaders/config-loader.ts | 126 +++++++- .../evaluation/validation/config-validator.ts | 98 ++++++ packages/core/src/index.ts | 11 + .../test/evaluation/env-injection.test.ts | 285 ++++++++++++++++++ .../evaluation/loaders/config-loader.test.ts | 132 ++++++++ .../validation/config-validator.test.ts | 103 +++++++ 9 files changed, 1030 insertions(+), 7 deletions(-) create mode 100644 packages/core/src/evaluation/env-injection.ts create mode 100644 packages/core/test/evaluation/env-injection.test.ts diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index f60968e23..8bc82505d 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -1,5 +1,10 @@ import path from 'node:path'; -import { loadConfig, runBeforeSessionHook } from '@agentv/core'; +import { + loadConfig, + loadEnvPathFiles, + runBeforeSessionHook, + runEnvFromEntries, +} from '@agentv/core'; import { binary, run, subcommands } from 'cmd-ts'; import { findRepoRoot } from './commands/eval/shared.js'; @@ -170,11 +175,21 @@ export async function runCli(argv: string[] = process.argv): Promise { } if (shouldRunBeforeSessionHook(processedArgv)) { - // Run before_session hook once at startup, before any command executes. - // Uses cwd as the search root for .agentv/config.yaml. + // Load env_path/env_from and run the before_session hook once at startup, + // before any command executes. Uses cwd as the search root for + // .agentv/config.yaml so validate/eval commands see the injected vars. const cwd = process.cwd(); const repoRoot = await findRepoRoot(cwd); const sessionConfig = await loadConfig(path.join(cwd, '_'), repoRoot); + const configDir = sessionConfig?.configDir ?? repoRoot; + + if (sessionConfig?.env_path) { + await loadEnvPathFiles(sessionConfig.env_path, configDir); + } + if (sessionConfig?.env_from) { + await runEnvFromEntries(sessionConfig.env_from, { cwd: configDir }); + } + const beforeSessionCommand = sessionConfig?.hooks?.before_session; if (beforeSessionCommand) { runBeforeSessionHook(beforeSessionCommand); diff --git a/apps/web/src/content/docs/docs/next/targets/configuration.mdx b/apps/web/src/content/docs/docs/next/targets/configuration.mdx index fdd83fa67..26a79ce17 100644 --- a/apps/web/src/content/docs/docs/next/targets/configuration.mdx +++ b/apps/web/src/content/docs/docs/next/targets/configuration.mdx @@ -193,6 +193,53 @@ targets: This keeps secrets out of version-controlled files and avoids requiring a CI step that rewrites already-exported secrets into `.env`. +### Loading Environment Variables from Config + +Set `env_path` and/or `env_from` in `.agentv/config.yaml` to load environment variables before +validation and eval run, so `{{ env.* }}` references resolve for both `agentv validate` and +`agentv eval`: + +```yaml +# .agentv/config.yaml +env_path: + - .env + - .env.local + +env_from: + - command: ["bun", "scripts/load-secrets.ts"] + format: shell_exports + - command: ["node", "scripts/print-env-json.mjs"] + format: json +``` + +- `env_path` loads one or more dotenv-style files, relative to the project directory (the parent + of `.agentv/`) unless the path is absolute. A missing file logs a warning and is skipped. +- `env_from` runs one or more argv commands and injects their parsed stdout. `command` must be a + non-empty argv array — shell command strings are not accepted. `format` defaults to + `shell_exports` (lines like `export KEY=value` or `KEY=value`); `format: json` expects a flat + JSON object of string values. A failing command aborts the CLI invocation with an error. +- Both singular (`env_path: .env`, `env_from: {command: [...]}`) and list forms are accepted. +- Values already present in the process environment are never overwritten, and injected values + are never printed to logs. +- An argv command that explicitly invokes a shell, such as `["sh", "-c", "..."]`, is still + accepted — the requirement is an argv array, not a ban on shells. Only implicit shell command + strings (`command: "bun scripts/load-secrets.ts"`) are rejected. + +`hooks.before_session` still runs for backward compatibility but is deprecated; migrate its +command to `env_from` (or a `.env` file plus `env_path`) instead. + +**Known limitations:** + +- `env_path`/`env_from` are discovered from the current working directory at CLI startup, the + same as legacy `hooks.before_session`. In a multi-project workspace, a nested project's own + `.agentv/config.yaml` `env_from` only runs when a command is invoked from within (or below) + that project's directory, not when run from a parent directory that resolves to a different + `.agentv/config.yaml`. +- A `.agentv/config.yaml`'s own fields (for example `results.repo`) are interpolated before that + same file's `env_path`/`env_from` runs, so they cannot reference values produced by its own + `env_path`/`env_from`. Use an already-exported process environment variable, or a value loaded + by an ancestor project's config, for fields inside `config.yaml` itself. + ## Supported Providers | Provider | Type | Description | diff --git a/packages/core/src/evaluation/env-injection.ts b/packages/core/src/evaluation/env-injection.ts new file mode 100644 index 000000000..f0644ec23 --- /dev/null +++ b/packages/core/src/evaluation/env-injection.ts @@ -0,0 +1,214 @@ +/** + * v5-native environment injection for AgentV project config. + * + * `env_path` loads dotenv-style files and `env_from` runs argv commands and + * parses their stdout, both injecting into `process.env` before validation + * and eval so target `{{ env.* }}` interpolation can see the values. This is + * the replacement path for the deprecated `hooks.before_session`. + * + * Existing `process.env` values always win — neither source overwrites a key + * that is already set. Values are never printed; only file paths, commands, + * and counts are logged. + * + * @module + */ + +import { spawnSync } from 'node:child_process'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +import type { EnvFromEntry, EnvFromFormat } from './loaders/config-loader.js'; + +const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; + +/** + * Parse `KEY=value` / `export KEY=value` lines, shared by `env_path` dotenv + * files and `env_from` `shell_exports` output. Quotes are stripped; blank + * lines, comments, and non-matching lines are skipped. + */ +export function parseShellExportsEnv(content: string): Record { + const result: Record = {}; + + for (const line of content.split('\n')) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith('#')) continue; + + const match = trimmed.match(/^(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)=(.*)$/); + if (!match) continue; + + const key = match[1]; + let value = match[2]; + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + result[key] = value; + } + + return result; +} + +/** + * Parse a flat JSON object of string values for `env_from` `format: json`. + * Throws on invalid JSON, non-object shapes, invalid env var names, or + * non-string values. + */ +export function parseJsonEnv(content: string): Record { + let parsed: unknown; + try { + parsed = JSON.parse(content); + } catch { + // Do not include the parser's message: JS engines embed a snippet of the + // offending content in JSON syntax errors, which could leak a secret + // value from malformed env_from output. + throw new Error('invalid JSON'); + } + + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('expected a flat JSON object of string values'); + } + + const result: Record = {}; + for (const [key, value] of Object.entries(parsed as Record)) { + if (!ENV_NAME_PATTERN.test(key)) { + throw new Error(`invalid environment variable name "${key}"`); + } + if (typeof value !== 'string') { + throw new Error(`value for "${key}" must be a string`); + } + result[key] = value; + } + + return result; +} + +/** Injects vars into process.env; existing keys and invalid names are skipped. Returns injected count. */ +function injectEnv(vars: Record): number { + let injected = 0; + for (const [key, value] of Object.entries(vars)) { + if (!ENV_NAME_PATTERN.test(key)) continue; + if (process.env[key] === undefined) { + process.env[key] = value; + injected++; + } + } + return injected; +} + +export type EnvPathLoadResult = { + readonly loaded: readonly string[]; + readonly missing: readonly string[]; + readonly injectedCount: number; +}; + +/** + * Load one or more dotenv-style files and inject their variables into + * `process.env`. Relative paths resolve against `baseDir`. A missing file + * warns and is skipped rather than failing the command. + */ +export async function loadEnvPathFiles( + envPaths: readonly string[], + baseDir: string, +): Promise { + const loaded: string[] = []; + const missing: string[] = []; + let injectedCount = 0; + + for (const envPath of envPaths) { + const resolvedPath = path.isAbsolute(envPath) ? envPath : path.join(baseDir, envPath); + + let content: string; + try { + content = await readFile(resolvedPath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + missing.push(resolvedPath); + logWarning(`env_path file not found: ${resolvedPath}`); + continue; + } + throw new Error(`Could not read env_path file ${resolvedPath}: ${(error as Error).message}`); + } + + injectedCount += injectEnv(parseShellExportsEnv(content)); + loaded.push(resolvedPath); + } + + if (injectedCount > 0) { + console.log( + `env_path injected ${injectedCount} environment variable(s) from ${loaded.length} file(s).`, + ); + } + + return { loaded, missing, injectedCount }; +} + +export type EnvFromRunResult = { + readonly injectedCount: number; +}; + +function parseEnvFromOutput(stdout: string, format: EnvFromFormat): Record { + return format === 'json' ? parseJsonEnv(stdout) : parseShellExportsEnv(stdout); +} + +/** + * Run one or more `env_from` argv commands and inject their parsed stdout + * into `process.env`. A non-zero exit, spawn failure, or unparseable output + * throws — command failures must fail the invoking command. stdout is never + * logged or included in error messages since it may carry secret values. + */ +export async function runEnvFromEntries( + entries: readonly EnvFromEntry[], + options: { readonly cwd: string }, +): Promise { + let injectedCount = 0; + + for (const entry of entries) { + const [command, ...args] = entry.command; + const commandLabel = entry.command.join(' '); + console.log(`Running env_from command: ${commandLabel}`); + + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + + if (result.stderr) { + process.stderr.write(result.stderr); + } + + if (result.error) { + throw new Error(`env_from command failed to start: ${commandLabel}: ${result.error.message}`); + } + if (result.status !== 0) { + throw new Error( + `env_from command exited with code ${result.status ?? 'unknown'}: ${commandLabel}`, + ); + } + + const format = entry.format ?? 'shell_exports'; + let vars: Record; + try { + vars = parseEnvFromOutput(result.stdout ?? '', format); + } catch (error) { + throw new Error( + `env_from command produced invalid ${format} output: ${commandLabel}: ${(error as Error).message}`, + ); + } + + injectedCount += injectEnv(vars); + } + + if (injectedCount > 0) { + console.log(`env_from injected ${injectedCount} environment variable(s).`); + } + + return { injectedCount }; +} + +function logWarning(message: string): void { + console.warn(`Warning: ${message}`); +} diff --git a/packages/core/src/evaluation/loaders/config-loader.ts b/packages/core/src/evaluation/loaders/config-loader.ts index 5f6f06f73..58279ef4f 100644 --- a/packages/core/src/evaluation/loaders/config-loader.ts +++ b/packages/core/src/evaluation/loaders/config-loader.ts @@ -62,10 +62,25 @@ export type ResultsConfig = { }; export type HooksConfig = { - /** Shell command to run once at agentv startup. stdout is parsed for env var exports. */ + /** + * Shell command to run once at agentv startup. stdout is parsed for env var + * exports. + * + * @deprecated Use `env_path` and/or `env_from` instead. `before_session` + * keeps running for now but will be removed in a future breaking release. + */ readonly before_session?: string; }; +export type EnvFromFormat = 'shell_exports' | 'json'; + +export type EnvFromEntry = { + /** Argv command array. Shell command strings are not accepted. */ + readonly command: readonly string[]; + /** Defaults to `shell_exports` when omitted. */ + readonly format?: EnvFromFormat; +}; + export type ReferenceMap = Readonly>; export type AgentVConfig = { @@ -74,6 +89,10 @@ export type AgentVConfig = { readonly execution?: ExecutionDefaults; readonly results?: ResultsConfig; readonly hooks?: HooksConfig; + /** Dotenv file(s) loaded before validation/eval, relative to `configDir` unless absolute. */ + readonly env_path?: readonly string[]; + /** Argv commands run before validation/eval to inject environment variables. */ + readonly env_from?: readonly EnvFromEntry[]; readonly refs?: ReferenceMap; /** * Promptfoo-shaped tags map applied to every run. Merged between eval `tags` @@ -81,6 +100,8 @@ export type AgentVConfig = { * reserved key `experiment` participates in experiment-namespace resolution. */ readonly tags?: Record; + /** Project directory containing `.agentv/`, for resolving relative `env_path` entries and `env_from` cwd. */ + readonly configDir?: string; } & ComposableConfigGraph; /** @@ -112,11 +133,11 @@ export async function loadConfig( continue; } - return readConfigFilePair(configPath, repoRoot); + return readConfigFilePair(configPath, repoRoot, directory); } return (await configPairExists(globalConfigPath)) - ? readConfigFilePair(globalConfigPath, repoRoot) + ? readConfigFilePair(globalConfigPath, repoRoot, getAgentvConfigDir()) : null; } @@ -148,6 +169,7 @@ async function readConfigObjectFile( async function readConfigFilePair( configPath: string, repoRoot: string, + projectDir: string, ): Promise { const localConfigPath = getLocalConfigPath(configPath); const base = stripLocalOnlyExecutionDefaults( @@ -165,7 +187,7 @@ async function readConfigFilePair( if (!rawMerged) { return null; } - return parseConfigObject(rawMerged, local ? localConfigPath : configPath, repoRoot); + return parseConfigObject(rawMerged, local ? localConfigPath : configPath, repoRoot, projectDir); } async function resolveConfigObjectFileReferences( @@ -182,6 +204,7 @@ function parseConfigObject( rawConfig: Record, configPath: string, repoRoot: string, + projectDir: string, ): AgentVConfig | null { try { const parsed = interpolateEnv(rawConfig, createEvalConfigEnv(repoRoot)) as unknown; @@ -221,6 +244,8 @@ function parseConfigObject( const executionDefaults = parseExecutionDefaults(rawExecution, configPath); const results = parseResultsConfig((parsed as Record).results, configPath); const hooks = parseHooksConfig((parsed as Record).hooks, configPath); + const envPath = parseEnvPathConfig((parsed as Record).env_path, configPath); + const envFrom = parseEnvFromConfig((parsed as Record).env_from, configPath); const tags = parseTagsConfig((parsed as Record).tags, configPath); const refs = parseRefsConfig((parsed as Record).refs, configPath); const graph = normalizeComposableConfigGraph(parsed as Record, configPath, { @@ -234,12 +259,15 @@ function parseConfigObject( ...(execution && { execution }), results, ...(hooks && { hooks }), + ...(envPath && { env_path: envPath }), + ...(envFrom && { env_from: envFrom }), ...(refs && { refs }), ...(tags && { tags }), ...(graph.targets && { targets: graph.targets }), ...(graph.graders && { graders: graph.graders }), ...(graph.tests && { tests: graph.tests }), ...(graph.defaults && { defaults: graph.defaults }), + configDir: projectDir, }; } catch (error) { const message = (error as Error).message; @@ -929,12 +957,102 @@ export function parseHooksConfig(raw: unknown, configPath: string): HooksConfig logWarning(`Invalid hooks.before_session in ${configPath}, expected non-empty string`); return undefined; } + logWarning( + `hooks.before_session in ${configPath} is deprecated; use env_path and/or env_from instead. before_session will keep running for now.`, + ); return { before_session: beforeSession.trim() }; } return undefined; } +const ENV_FROM_FORMATS: ReadonlySet = new Set(['shell_exports', 'json']); + +/** + * Parse the `env_path` field from .agentv/config.yaml. + * Accepts a single string or an array of strings; invalid entries are dropped with a warning. + */ +export function parseEnvPathConfig( + raw: unknown, + configPath: string, +): readonly string[] | undefined { + if (raw === undefined || raw === null) { + return undefined; + } + + const entries = Array.isArray(raw) ? raw : [raw]; + const paths: string[] = []; + for (const entry of entries) { + if (typeof entry !== 'string' || entry.trim().length === 0) { + logWarning(`Invalid env_path entry in ${configPath}, expected non-empty string`); + continue; + } + paths.push(entry.trim()); + } + + return paths.length > 0 ? paths : undefined; +} + +/** + * Parse the `env_from` field from .agentv/config.yaml. + * Accepts a single entry object or an array of entry objects. Each entry + * requires a non-empty argv `command` array; shell command strings are + * rejected. `format` defaults to `shell_exports`. Invalid entries are dropped + * with a warning. + */ +export function parseEnvFromConfig( + raw: unknown, + configPath: string, +): readonly EnvFromEntry[] | undefined { + if (raw === undefined || raw === null) { + return undefined; + } + + const isList = Array.isArray(raw); + const entries = isList ? raw : [raw]; + const result: EnvFromEntry[] = []; + + entries.forEach((entry, index) => { + const location = isList ? `env_from[${index}]` : 'env_from'; + if (!isJsonObject(entry)) { + logWarning(`Invalid ${location} in ${configPath}, expected object`); + return; + } + + const rawCommand = entry.command; + if (typeof rawCommand === 'string') { + logWarning( + `Invalid ${location}.command in ${configPath}: shell command strings are not supported, use an argv array such as ["bun", "scripts/load-secrets.ts"]`, + ); + return; + } + if ( + !Array.isArray(rawCommand) || + rawCommand.length === 0 || + !rawCommand.every((part) => typeof part === 'string' && part.length > 0) + ) { + logWarning(`Invalid ${location}.command in ${configPath}, expected a non-empty string array`); + return; + } + + const rawFormat = entry.format; + let format: EnvFromFormat = 'shell_exports'; + if (rawFormat !== undefined) { + if (typeof rawFormat !== 'string' || !ENV_FROM_FORMATS.has(rawFormat as EnvFromFormat)) { + logWarning( + `Invalid ${location}.format in ${configPath}, expected "shell_exports" or "json"`, + ); + return; + } + format = rawFormat as EnvFromFormat; + } + + result.push({ command: rawCommand as readonly string[], format }); + }); + + return result.length > 0 ? result : undefined; +} + /** * Parse the optional project-config `tags` map (promptfoo-shaped * `Record`). Non-string entries are dropped with a warning; a diff --git a/packages/core/src/evaluation/validation/config-validator.ts b/packages/core/src/evaluation/validation/config-validator.ts index 95bb42d2a..02fb0e277 100644 --- a/packages/core/src/evaluation/validation/config-validator.ts +++ b/packages/core/src/evaluation/validation/config-validator.ts @@ -93,6 +93,9 @@ export async function validateConfigFile( validateRefsConfig(errors, filePath, config.refs); validateComposableGraph(errors, filePath, config); validateDashboardConfig(errors, filePath, config.dashboard); + validateHooksConfig(errors, filePath, config.hooks); + validateEnvPathConfig(errors, filePath, config.env_path); + validateEnvFromConfig(errors, filePath, config.env_from); const projects = config.projects; if (projects !== undefined) { @@ -131,6 +134,9 @@ export async function validateConfigFile( 'projects', 'dashboard', 'studio', + 'hooks', + 'env_path', + 'env_from', ]); const unexpectedFields = Object.keys(config).filter((key) => !allowedFields.has(key)); @@ -192,6 +198,98 @@ function validateDashboardConfig( } } +const ENV_FROM_FORMATS = new Set(['shell_exports', 'json']); + +function validateHooksConfig(errors: ValidationError[], filePath: string, rawHooks: unknown): void { + if (rawHooks === undefined || rawHooks === null) { + return; + } + if (!isPlainObject(rawHooks)) { + addError(errors, filePath, 'hooks', "Field 'hooks' must be an object"); + return; + } + + if (rawHooks.before_session !== undefined) { + validateOptionalString(errors, filePath, rawHooks.before_session, 'hooks.before_session'); + errors.push({ + severity: 'warning', + filePath, + location: 'hooks.before_session', + message: + "'hooks.before_session' is deprecated; use 'env_path' and/or 'env_from' to load environment variables before validation and eval.", + }); + } +} + +function validateEnvPathConfig( + errors: ValidationError[], + filePath: string, + rawEnvPath: unknown, +): void { + if (rawEnvPath === undefined || rawEnvPath === null) { + return; + } + + const isList = Array.isArray(rawEnvPath); + const entries = isList ? rawEnvPath : [rawEnvPath]; + entries.forEach((entry, index) => { + const location = isList ? `env_path[${index}]` : 'env_path'; + if (typeof entry !== 'string' || entry.trim().length === 0) { + addError(errors, filePath, location, `Field '${location}' must be a non-empty string`); + } + }); +} + +function validateEnvFromConfig( + errors: ValidationError[], + filePath: string, + rawEnvFrom: unknown, +): void { + if (rawEnvFrom === undefined || rawEnvFrom === null) { + return; + } + + const isList = Array.isArray(rawEnvFrom); + const entries = isList ? rawEnvFrom : [rawEnvFrom]; + entries.forEach((entry, index) => { + const location = isList ? `env_from[${index}]` : 'env_from'; + if (!isPlainObject(entry)) { + addError(errors, filePath, location, `Field '${location}' must be an object`); + return; + } + + const command = entry.command; + if (typeof command === 'string') { + addError( + errors, + filePath, + `${location}.command`, + `Field '${location}.command' must be an argv array of strings, not a shell command string. Use e.g. ["bun", "scripts/load-secrets.ts"].`, + ); + } else if ( + !Array.isArray(command) || + command.length === 0 || + !command.every((part) => typeof part === 'string' && part.length > 0) + ) { + addError( + errors, + filePath, + `${location}.command`, + `Field '${location}.command' must be a non-empty array of strings`, + ); + } + + if (entry.format !== undefined && !ENV_FROM_FORMATS.has(entry.format as string)) { + addError( + errors, + filePath, + `${location}.format`, + `Field '${location}.format' must be "shell_exports" or "json"`, + ); + } + }); +} + function validateRefsConfig(errors: ValidationError[], filePath: string, rawRefs: unknown): void { if (rawRefs === undefined) { return; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index cdaa11bc9..b255e8631 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -24,6 +24,9 @@ export { resolveResultsConfigForProject, type AgentVConfig as AgentVYamlConfig, type ResultsConfig, + type HooksConfig, + type EnvFromEntry, + type EnvFromFormat, } from './evaluation/loaders/config-loader.js'; export { loadTsEvalFile, @@ -238,6 +241,14 @@ export { export { discoverGraders } from './evaluation/registry/grader-discovery.js'; export { RunBudgetTracker } from './evaluation/run-budget-tracker.js'; export { runBeforeSessionHook, parseEnvOutput } from './evaluation/hooks.js'; +export { + loadEnvPathFiles, + runEnvFromEntries, + parseShellExportsEnv, + parseJsonEnv, + type EnvPathLoadResult, + type EnvFromRunResult, +} from './evaluation/env-injection.js'; export { trackChild, killAllTrackedChildren, diff --git a/packages/core/test/evaluation/env-injection.test.ts b/packages/core/test/evaluation/env-injection.test.ts new file mode 100644 index 000000000..65e746ffc --- /dev/null +++ b/packages/core/test/evaluation/env-injection.test.ts @@ -0,0 +1,285 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; + +import { + loadEnvPathFiles, + parseJsonEnv, + parseShellExportsEnv, + runEnvFromEntries, +} from '../../src/evaluation/env-injection.js'; +import { resolveTargetDefinition } from '../../src/evaluation/providers/targets.js'; + +describe('parseShellExportsEnv', () => { + it('parses export and bare KEY=value lines, stripping quotes', () => { + const input = ['export FOO="bar"', "BAZ='qux'", 'QUUX=plain'].join('\n'); + expect(parseShellExportsEnv(input)).toEqual({ FOO: 'bar', BAZ: 'qux', QUUX: 'plain' }); + }); + + it('ignores blank lines and comments', () => { + expect(parseShellExportsEnv('\n# comment\nFOO=bar\n\n')).toEqual({ FOO: 'bar' }); + }); + + it('ignores lines with invalid env var names', () => { + expect(parseShellExportsEnv('123FOO=bar\nVALID=ok')).toEqual({ VALID: 'ok' }); + }); +}); + +describe('parseJsonEnv', () => { + it('parses a flat JSON object of string values', () => { + expect(parseJsonEnv('{"FOO":"bar","BAZ":"qux"}')).toEqual({ FOO: 'bar', BAZ: 'qux' }); + }); + + it('rejects invalid JSON', () => { + expect(() => parseJsonEnv('not json')).toThrow(/invalid JSON/); + }); + + it('rejects non-object JSON shapes', () => { + expect(() => parseJsonEnv('["FOO"]')).toThrow(/flat JSON object/); + }); + + it('rejects non-string values', () => { + expect(() => parseJsonEnv('{"FOO":1}')).toThrow(/must be a string/); + }); + + it('rejects invalid environment variable names', () => { + expect(() => parseJsonEnv('{"123FOO":"bar"}')).toThrow(/invalid environment variable name/); + }); + + it('never leaks a fragment of malformed content in the JSON parse error', () => { + try { + parseJsonEnv('{"API_KEY": sk_live_SUPER_SECRET_TOKEN_123}'); + throw new Error('expected parseJsonEnv to throw'); + } catch (error) { + expect((error as Error).message).not.toContain('SUPER_SECRET_TOKEN'); + expect((error as Error).message).toBe('invalid JSON'); + } + }); +}); + +describe('loadEnvPathFiles', () => { + let tempDir: string; + const testKeys = ['AGENTV_TEST_ENV_PATH_NEW', 'AGENTV_TEST_ENV_PATH_EXISTING']; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(os.tmpdir(), 'agentv-env-path-')); + for (const key of testKeys) delete process.env[key]; + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + for (const key of testKeys) delete process.env[key]; + }); + + it('injects missing env vars from a dotenv file relative to baseDir', async () => { + await writeFile(path.join(tempDir, '.env'), 'AGENTV_TEST_ENV_PATH_NEW=from-file\n'); + + const result = await loadEnvPathFiles(['.env'], tempDir); + + expect(process.env.AGENTV_TEST_ENV_PATH_NEW).toBe('from-file'); + expect(result.injectedCount).toBe(1); + expect(result.missing).toEqual([]); + }); + + it('does not override an existing process.env value', async () => { + process.env.AGENTV_TEST_ENV_PATH_EXISTING = 'from-process'; + await writeFile(path.join(tempDir, '.env'), 'AGENTV_TEST_ENV_PATH_EXISTING=from-file\n'); + + const result = await loadEnvPathFiles(['.env'], tempDir); + + expect(process.env.AGENTV_TEST_ENV_PATH_EXISTING).toBe('from-process'); + expect(result.injectedCount).toBe(0); + }); + + it('warns and continues when a file is missing instead of failing', async () => { + const result = await loadEnvPathFiles(['.env.missing'], tempDir); + + expect(result.missing).toHaveLength(1); + expect(result.loaded).toEqual([]); + }); +}); + +describe('runEnvFromEntries', () => { + const testKeys = [ + 'AGENTV_TEST_ENV_FROM_SHELL', + 'AGENTV_TEST_ENV_FROM_JSON', + 'AGENTV_TEST_ENV_FROM_EXISTING', + ]; + + beforeEach(() => { + for (const key of testKeys) delete process.env[key]; + }); + + afterEach(() => { + for (const key of testKeys) delete process.env[key]; + }); + + it('injects env vars parsed from shell_exports command output', async () => { + const result = await runEnvFromEntries( + [ + { + command: ['bun', '-e', "process.stdout.write('export AGENTV_TEST_ENV_FROM_SHELL=hi\\n')"], + format: 'shell_exports', + }, + ], + { cwd: process.cwd() }, + ); + + expect(process.env.AGENTV_TEST_ENV_FROM_SHELL).toBe('hi'); + expect(result.injectedCount).toBe(1); + }); + + it('injects env vars parsed from json command output', async () => { + await runEnvFromEntries( + [ + { + command: [ + 'bun', + '-e', + "process.stdout.write(JSON.stringify({AGENTV_TEST_ENV_FROM_JSON: 'value'}))", + ], + format: 'json', + }, + ], + { cwd: process.cwd() }, + ); + + expect(process.env.AGENTV_TEST_ENV_FROM_JSON).toBe('value'); + }); + + it('does not override an existing process.env value', async () => { + process.env.AGENTV_TEST_ENV_FROM_EXISTING = 'from-process'; + + const result = await runEnvFromEntries( + [ + { + command: [ + 'bun', + '-e', + "process.stdout.write('AGENTV_TEST_ENV_FROM_EXISTING=from-command\\n')", + ], + }, + ], + { cwd: process.cwd() }, + ); + + expect(process.env.AGENTV_TEST_ENV_FROM_EXISTING).toBe('from-process'); + expect(result.injectedCount).toBe(0); + }); + + it('throws a useful error when the command exits non-zero', async () => { + await expect( + runEnvFromEntries([{ command: ['bun', '-e', 'process.exit(1)'] }], { cwd: process.cwd() }), + ).rejects.toThrow(/exited with code 1/); + }); + + it('throws when json output is invalid', async () => { + await expect( + runEnvFromEntries( + [{ command: ['bun', '-e', "process.stdout.write('not json')"], format: 'json' }], + { cwd: process.cwd() }, + ), + ).rejects.toThrow(/invalid json output/); + }); + + it('never leaks malformed JSON command output in the thrown error', async () => { + // The secret must live only in the script's runtime stdout, not in the + // command's own argv, so this isolates the "stdout leaking into an error + // message" vector from the (expected, non-secret) command-argv logging. + const tempDir = await mkdtemp(path.join(os.tmpdir(), 'agentv-env-from-leak-')); + try { + const scriptPath = path.join(tempDir, 'print-bad-json.ts'); + await writeFile( + scriptPath, + 'process.stdout.write(String.raw`{"API_KEY": sk_live_SUPER_SECRET_TOKEN_123}`)', + ); + + try { + await runEnvFromEntries([{ command: ['bun', scriptPath], format: 'json' }], { + cwd: process.cwd(), + }); + throw new Error('expected runEnvFromEntries to throw'); + } catch (error) { + expect((error as Error).message).not.toContain('SUPER_SECRET_TOKEN'); + } + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); +}); + +describe('env injection feeds {{ env.* }} target interpolation', () => { + beforeEach(() => { + process.env.AGENTV_TEST_MODEL = 'gpt-5-mini'; + }); + + afterEach(() => { + process.env.AGENTV_TEST_SECRET_FROM_ENV_PATH = undefined; + process.env.AGENTV_TEST_SECRET_FROM_ENV_FROM = undefined; + process.env.AGENTV_TEST_MODEL = undefined; + }); + + it('resolves a target field from a value injected via env_path', async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), 'agentv-env-path-target-')); + try { + await writeFile( + path.join(tempDir, '.env'), + 'AGENTV_TEST_SECRET_FROM_ENV_PATH=from-env-path\n', + ); + + await loadEnvPathFiles(['.env'], tempDir); + + const target = resolveTargetDefinition( + { + id: 'oracle', + provider: 'openai', + config: { + api_key: '{{ env.AGENTV_TEST_SECRET_FROM_ENV_PATH }}', + model: '{{ env.AGENTV_TEST_MODEL }}', + }, + } as never, + process.env, + ); + + if (target.kind !== 'openai') { + throw new Error('expected openai target'); + } + expect(target.config.apiKey).toBe('from-env-path'); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); + + it('resolves a target field from a value injected via env_from', async () => { + await runEnvFromEntries( + [ + { + command: [ + 'bun', + '-e', + "process.stdout.write('export AGENTV_TEST_SECRET_FROM_ENV_FROM=from-env-from\\n')", + ], + }, + ], + { cwd: process.cwd() }, + ); + + const target = resolveTargetDefinition( + { + id: 'oracle', + provider: 'openai', + config: { + api_key: '{{ env.AGENTV_TEST_SECRET_FROM_ENV_FROM }}', + model: '{{ env.AGENTV_TEST_MODEL }}', + }, + } as never, + process.env, + ); + + if (target.kind !== 'openai') { + throw new Error('expected openai target'); + } + expect(target.config.apiKey).toBe('from-env-from'); + }); +}); diff --git a/packages/core/test/evaluation/loaders/config-loader.test.ts b/packages/core/test/evaluation/loaders/config-loader.test.ts index 41f5b2a21..a9a814b0b 100644 --- a/packages/core/test/evaluation/loaders/config-loader.test.ts +++ b/packages/core/test/evaluation/loaders/config-loader.test.ts @@ -13,7 +13,10 @@ import { extractThreshold, extractWorkersFromSuite, loadConfig, + parseEnvFromConfig, + parseEnvPathConfig, parseExecutionDefaults, + parseHooksConfig, parseResultsConfig, parseTagsConfig, resolveResultsConfigForProject, @@ -1202,3 +1205,132 @@ describe('parseExecutionDefaults', () => { ).toThrow(/execution\.otel_backend/); }); }); + +describe('parseHooksConfig', () => { + it('logs a deprecation warning and still parses before_session', () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + try { + const result = parseHooksConfig({ before_session: 'echo hi' }, '/test/config.yaml'); + expect(result).toEqual({ before_session: 'echo hi' }); + expect(warnSpy.mock.calls.some(([msg]) => String(msg).includes('deprecated'))).toBe(true); + } finally { + warnSpy.mockRestore(); + } + }); + + it('returns undefined when before_session is absent', () => { + expect(parseHooksConfig({}, '/test/config.yaml')).toBeUndefined(); + }); +}); + +describe('parseEnvPathConfig', () => { + it('normalizes a singular string into an array', () => { + expect(parseEnvPathConfig('.env', '/test/config.yaml')).toEqual(['.env']); + }); + + it('accepts an array of strings', () => { + expect(parseEnvPathConfig(['.env', '.env.local'], '/test/config.yaml')).toEqual([ + '.env', + '.env.local', + ]); + }); + + it('drops non-string entries with a warning and returns undefined when none remain', () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + try { + expect(parseEnvPathConfig(42, '/test/config.yaml')).toBeUndefined(); + } finally { + warnSpy.mockRestore(); + } + }); +}); + +describe('parseEnvFromConfig', () => { + it('normalizes a singular object into an array and defaults format', () => { + const result = parseEnvFromConfig( + { command: ['bun', 'scripts/load-secrets.ts'] }, + '/test/config.yaml', + ); + expect(result).toEqual([ + { command: ['bun', 'scripts/load-secrets.ts'], format: 'shell_exports' }, + ]); + }); + + it('accepts an array of entries with explicit formats', () => { + const result = parseEnvFromConfig( + [ + { command: ['bun', 'scripts/load-secrets.ts'], format: 'shell_exports' }, + { command: ['node', 'scripts/print-env-json.mjs'], format: 'json' }, + ], + '/test/config.yaml', + ); + expect(result).toEqual([ + { command: ['bun', 'scripts/load-secrets.ts'], format: 'shell_exports' }, + { command: ['node', 'scripts/print-env-json.mjs'], format: 'json' }, + ]); + }); + + it('rejects a shell command string and drops the entry', () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + try { + const result = parseEnvFromConfig( + { command: 'bun scripts/load-secrets.ts' }, + '/test/config.yaml', + ); + expect(result).toBeUndefined(); + expect( + warnSpy.mock.calls.some(([msg]) => + String(msg).includes('shell command strings are not supported'), + ), + ).toBe(true); + } finally { + warnSpy.mockRestore(); + } + }); + + it('rejects an invalid format value', () => { + const warnSpy = spyOn(console, 'warn').mockImplementation(() => {}); + try { + const result = parseEnvFromConfig( + { command: ['bun', 'x.ts'], format: 'yaml' }, + '/test/config.yaml', + ); + expect(result).toBeUndefined(); + } finally { + warnSpy.mockRestore(); + } + }); +}); + +describe('loadConfig env_path / env_from / configDir', () => { + it('parses env_path and env_from and exposes the project directory (not .agentv/) as configDir', async () => { + const tempDir = mkdtempSync(path.join(os.tmpdir(), 'agentv-config-env-')); + try { + const projectDir = path.join(tempDir, 'project'); + const dotAgentvDir = path.join(projectDir, '.agentv'); + mkdirSync(dotAgentvDir, { recursive: true }); + writeFileSync( + path.join(dotAgentvDir, 'config.yaml'), + [ + 'env_path:', + ' - .env', + ' - .env.local', + 'env_from:', + ' - command: ["bun", "scripts/load-secrets.ts"]', + '', + ].join('\n'), + ); + + const config = await loadConfig(path.join(projectDir, 'evals', '_'), projectDir); + + expect(config?.env_path).toEqual(['.env', '.env.local']); + expect(config?.env_from).toEqual([ + { command: ['bun', 'scripts/load-secrets.ts'], format: 'shell_exports' }, + ]); + // env_path files such as `.env` sit beside `.agentv/`, at the project root. + expect(config?.configDir).toBe(projectDir); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/core/test/evaluation/validation/config-validator.test.ts b/packages/core/test/evaluation/validation/config-validator.test.ts index 56db4911f..cdec5eadd 100644 --- a/packages/core/test/evaluation/validation/config-validator.test.ts +++ b/packages/core/test/evaluation/validation/config-validator.test.ts @@ -552,4 +552,107 @@ describe('validateConfigFile', () => { const warnings = result.errors.filter((e) => e.severity === 'warning'); expect(warnings.some((e) => e.message.includes('Unexpected fields: foo'))).toBe(true); }); + + it('accepts hooks.before_session with a deprecation warning, not an unexpected field', async () => { + const filePath = path.join(tempDir, 'config-hooks.yaml'); + await writeFile( + filePath, + `hooks: + before_session: "bun scripts/load-secrets.ts" +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors.some((e) => e.message.includes('Unexpected fields'))).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ severity: 'warning', location: 'hooks.before_session' }), + ); + }); + + it('accepts env_path as a string or array without unexpected-field warnings', async () => { + const filePath = path.join(tempDir, 'config-env-path.yaml'); + await writeFile( + filePath, + `env_path: + - .env + - .env.local +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors.some((e) => e.message.includes('Unexpected fields'))).toBe(false); + }); + + it('accepts env_from as an object or array of argv command entries', async () => { + const filePath = path.join(tempDir, 'config-env-from.yaml'); + await writeFile( + filePath, + `env_from: + - command: ["bun", "scripts/load-secrets.ts"] + format: shell_exports + - command: ["node", "scripts/print-env-json.mjs"] + format: json +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('rejects env_from.command given as a shell string', async () => { + const filePath = path.join(tempDir, 'config-env-from-shell-string.yaml'); + await writeFile( + filePath, + `env_from: + command: "bun scripts/load-secrets.ts" +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ severity: 'error', location: 'env_from.command' }), + ); + }); + + it('rejects an invalid env_from.format value', async () => { + const filePath = path.join(tempDir, 'config-env-from-bad-format.yaml'); + await writeFile( + filePath, + `env_from: + command: ["bun", "x.ts"] + format: yaml +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(false); + expect(result.errors).toContainEqual( + expect.objectContaining({ severity: 'error', location: 'env_from.format' }), + ); + }); + + it('accepts empty env_path/env_from/hooks keys the same way the loader treats them as absent', async () => { + const filePath = path.join(tempDir, 'config-empty-env-keys.yaml'); + await writeFile( + filePath, + `env_path: +env_from: +hooks: +`, + ); + + const result = await validateConfigFile(filePath); + + expect(result.valid).toBe(true); + expect(result.errors).toHaveLength(0); + }); });