input.ts

/**
 * `agentv pipeline input` — Extract eval inputs, target invocation info, and grader
 * configurations for subagent-mode eval runs.
 *
 * Reads an eval YAML file and writes a structured export directory that agents
 * and Python wrapper scripts can consume without re-parsing YAML or resolving
 * file references.
 *
 * Export directory layout:
 *   <out-dir>/
 *   ├── manifest.json
 *   └── <suite>/                (omitted if eval.yaml has no name)
 *       └── <test-id>/
 *           ├── input.json
 *           ├── invoke.json
 *           ├── criteria.md
 *           ├── expected_output.json    (if present)
 *           ├── llm_graders/<name>.json
 *           └── script_graders/<name>.json   # script/deterministic grader configs
 */
import { readFile } from 'node:fs/promises';
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname, join, relative, resolve } from 'node:path';

import type { GraderConfig, LlmBackedGraderConfig, ScriptGraderConfig } from '@agentv/core';

/** Assertion types that can be graded deterministically without external scripts or LLMs. */
const BUILTIN_ASSERTION_TYPES = new Set([
  'contains',
  'contains-any',
  'contains-all',
  'icontains',
  'icontains-any',
  'icontains-all',
  'starts-with',
  'ends-with',
  'regex',
  'is-json',
  'equals',
]);
import { deriveCategory, loadTestSuite } from '@agentv/core';
import { command, option, optional, positional, string } from 'cmd-ts';

import { buildDefaultRunDir } from '../eval/result-layout.js';
import { findRepoRoot } from '../eval/shared.js';
import { selectTarget } from '../eval/targets.js';

export const evalInputCommand = command({
  name: 'input',
  description: 'Extract eval inputs, target commands, and grader prompts for subagent-mode runs',
  args: {
    evalPath: positional({
      type: string,
      displayName: 'eval-path',
      description: 'Path to eval YAML file',
    }),
    out: option({
      type: optional(string),
      long: 'out',
      description: 'Output directory for extracted inputs (default: .agentv/results/<run_id>)',
    }),
    experiment: option({
      type: optional(string),
      long: 'experiment',
      description: 'Experiment label (e.g. with_skills, without_skills)',
    }),
    target: option({
      type: optional(string),
      long: 'target',
      description: 'Override target name from targets.yaml (mirrors eval run --target)',
    }),
    targets: option({
      type: optional(string),
      long: 'targets',
      description: 'Path to targets.yaml (overrides discovery)',
    }),
  },
  handler: async ({ evalPath, out, experiment, target, targets }) => {
    const resolvedEvalPath = resolve(evalPath);
    const outDir = resolve(out ?? buildDefaultRunDir(process.cwd(), experiment));
    const repoRoot = await findRepoRoot(dirname(resolvedEvalPath));
    const evalDir = dirname(resolvedEvalPath);

    const category = deriveCategory(relative(process.cwd(), resolvedEvalPath));
    const suite = await loadTestSuite(resolvedEvalPath, repoRoot, { category });
    const tests = suite.tests;

    if (tests.length === 0) {
      console.error('No tests found in eval file.');
      process.exit(1);
    }

    // Try to resolve target for CLI invocation info.
    // Non-CLI providers default to agent mode (executor subagents) unless
    // subagent_mode_allowed: false is set in targets.yaml.
    let targetInfo: { kind: 'cli'; command: string; cwd: string; timeoutMs: number } | null = null;
    let targetName = 'agent';
    let targetKind = 'agent';
    let subagentModeAllowed = true;

    try {
      const selection = await selectTarget({
        testFilePath: resolvedEvalPath,
        repoRoot,
        cwd: evalDir,
        cliTargetName: target,
        explicitTargetsPath: targets,
        env: process.env,
      });

      targetName = selection.targetName;
      const resolved = selection.resolvedTarget;
      subagentModeAllowed = resolved.subagentModeAllowed !== false;

      if (resolved.kind === 'cli') {
        targetKind = 'cli';
        subagentModeAllowed = false;
        const config = resolved.config;
        targetInfo = {
          kind: 'cli',
          command: config.command,
          cwd: config.cwd ?? evalDir,
          timeoutMs: config.timeoutMs ?? 30000,
        };
      } else {
        targetKind = resolved.kind;
      }
    } catch {
      // No targets file found — subagent-as-target mode
    }

    // Use tests[0].suite — loaders (yaml-parser, jsonl-parser) already apply the
    // metadata.name → filename-basename → 'eval' fallback for subagent-mode labels.
    const suiteName = tests[0]?.suite?.trim() ?? '';
    const safeSuiteName = suiteName ? suiteName.replace(/[\/\\:*?"<>|]/g, '_') : '';

    const testIds: string[] = [];

    for (const test of tests) {
      const subpath = safeSuiteName ? [safeSuiteName, test.id] : [test.id];
      const testDir = join(outDir, ...subpath);
      await mkdir(testDir, { recursive: true });
      testIds.push(test.id);

      // input.json - aligned with eval YAML schema and script grader payload fields.
      const inputMessages = test.input.map((m) => ({
        role: m.role,
        content: typeof m.content === 'string' ? m.content : m.content,
      }));
      await writeJson(join(testDir, 'input.json'), {
        input: inputMessages,
        input_files: test.file_paths,
        ...(test.description ? { description: test.description } : {}),
        metadata: test.metadata ?? {},
      });

      // invoke.json — CLI targets get command info; non-CLI targets get agent mode.
      // The manifest carries subagent_mode_allowed so consumers (the bench skill)
      // can decide whether to dispatch executor subagents or use `agentv eval`.
      if (targetInfo) {
        await writeJson(join(testDir, 'invoke.json'), {
          kind: 'cli',
          command: targetInfo.command,
          cwd: targetInfo.cwd,
          timeout_ms: targetInfo.timeoutMs,
          env: {},
        });
      } else {
        await writeJson(join(testDir, 'invoke.json'), {
          kind: 'agent',
          instructions: 'Execute this task in the current workspace. The agent IS the target.',
        });
      }

      // criteria.md
      await writeFile(join(testDir, 'criteria.md'), test.criteria ?? '', 'utf8');

      // expected_output.json (if present)
      if (
        test.expected_output.length > 0 ||
        (test.reference_answer !== undefined && test.reference_answer !== '')
      ) {
        await writeJson(join(testDir, 'expected_output.json'), {
          expected_output: test.expected_output,
          reference_answer: test.reference_answer ?? '',
        });
      }

      // Grader configs
      await writeGraderConfigs(testDir, test.assertions ?? [], evalDir);
    }

    // manifest.json
    await writeJson(join(outDir, 'manifest.json'), {
      eval_file: resolvedEvalPath,
      suite: suiteName || undefined,
      experiment: experiment || undefined,
      timestamp: new Date().toISOString(),
      target: {
        name: targetName,
        kind: targetKind,
        subagent_mode_allowed: subagentModeAllowed,
      },
      test_ids: testIds,
    });

    console.log(`Extracted ${testIds.length} test(s) to ${outDir}`);

    // --- Subagent mode guidance ---
    if (targetKind === 'agent') {
      console.log(`
  Target: ${targetName} (subagent-as-target mode)`);
      console.log(`  Tests:  ${testIds.join(', ')}`);
      console.log('');
      console.log('  Next steps for the orchestrating agent:');
      console.log('  1. Dispatch executor subagents — one per test case (all in parallel):');
      console.log('     - Each reads <run-dir>/<test-id>/input.json');
      console.log('     - Executes the task, writes <run-dir>/<test-id>/response.md');
      console.log('  2. Run script graders: agentv pipeline grade <run-dir>');
      console.log(
        '  3. Dispatch grader subagents — one per (test × LLM grader) pair (all in parallel):',
      );
      console.log(
        '     - Read agents/grader.md and embed its content as system instructions in each subagent prompt',
      );
      console.log('     - Each subagent reads llm_graders/<name>.json + response.md for its test');
      console.log('     - Each writes llm_grader_results/<name>.json');
      console.log('  4. Merge scores:     agentv pipeline bench <run-dir>');
      console.log('');
      console.log('  For the full procedure:');
      console.log('    agentv skills get agentv-bench --ref subagent-pipeline');
      console.log('');
    }
  },
});

interface GraderCounts {
  scriptGraders: number;
  llmGraders: number;
  builtinAssertions: number;
}

async function writeGraderConfigs(
  testDir: string,
  assertions: readonly GraderConfig[],
  evalDir: string,
): Promise<GraderCounts> {
  const counts: GraderCounts = { scriptGraders: 0, llmGraders: 0, builtinAssertions: 0 };
  const scriptGradersDir = join(testDir, 'script_graders');
  const llmGradersDir = join(testDir, 'llm_graders');

  let hasScriptGraders = false;
  let hasLlmGraders = false;

  for (const assertion of assertions) {
    if (assertion.type === 'script') {
      if (!hasScriptGraders) {
        await mkdir(scriptGradersDir, { recursive: true });
        hasScriptGraders = true;
      }
      const config = assertion as ScriptGraderConfig;
      await writeJson(join(scriptGradersDir, `${config.name}.json`), {
        name: config.name,
        type: 'script',
        command: config.command,
        cwd: config.resolvedCwd ?? config.cwd ?? evalDir,
        weight: config.weight ?? 1.0,
        config: config.config ?? {},
      });
    } else if (assertion.type === 'llm-grader' || assertion.type === 'llm-rubric') {
      if (!hasLlmGraders) {
        await mkdir(llmGradersDir, { recursive: true });
        hasLlmGraders = true;
      }
      const config = assertion as LlmBackedGraderConfig;
      let promptContent = '';

      if (config.resolvedPromptPath) {
        try {
          promptContent = await readFile(config.resolvedPromptPath, 'utf8');
        } catch {
          promptContent = typeof config.prompt === 'string' ? config.prompt : '';
        }
      } else if (typeof config.prompt === 'string') {
        promptContent = config.prompt;
      }

      // For rubrics assertions, include the criteria array directly
      // so grader subagents can evaluate without needing a prompt file.
      const rubrics = config.rubrics;
      const rubricsData = rubrics?.map((r) => ({
        id: r.id,
        outcome: r.outcome,
        weight: r.weight ?? 1.0,
        ...(r.score_ranges ? { score_range: r.score_ranges } : {}),
        ...(r.required !== undefined ? { required: r.required } : {}),
        ...(r.min_score !== undefined ? { min_score: r.min_score } : {}),
      }));

      await writeJson(join(llmGradersDir, `${config.name}.json`), {
        name: config.name,
        type: config.type,
        prompt_content: promptContent,
        ...(config.type === 'llm-rubric' && config.value !== undefined
          ? { value: config.value }
          : {}),
        ...(rubricsData && rubricsData.length > 0 ? { rubrics: rubricsData } : {}),
        weight: config.weight ?? 1.0,
        threshold: 0.5,
        config: {},
      });
    } else if (BUILTIN_ASSERTION_TYPES.has(assertion.type)) {
      if (!hasScriptGraders) {
        await mkdir(scriptGradersDir, { recursive: true });
        hasScriptGraders = true;
      }
      const config = assertion as GraderConfig & { value?: unknown; flags?: string };
      await writeJson(join(scriptGradersDir, `${config.name}.json`), {
        name: config.name,
        type: config.type,
        value: config.value,
        flags: (config as { flags?: string }).flags,
        weight: config.weight ?? 1.0,
        negate: config.negate ?? false,
      });
    }
  }
  return counts;
}

async function writeJson(filePath: string, data: unknown): Promise<void> {
  await writeFile(filePath, `${JSON.stringify(data, null, 2)}\n`, 'utf8');
}