fix(ci): guard eval schema generation (#1725)

Author: christso

.agents/verification.md

@@ -12,6 +12,11 @@ This file expands [AGENTS.md](../AGENTS.md) for testing, manual UAT, CLI and bro
   smallest command that exercises the changed code, such as a package-specific
   test, a single test file, `bun run typecheck` for type-facing changes, or
   `bun run validate:examples` for eval example changes.
+- When changing `packages/core/src/evaluation/validation/eval-file.schema.ts`,
+  run `bun run validate:eval-schema` from the repo root. The command regenerates
+  `skills-data/agentv-eval-writer/references/eval.schema.json` from the Zod
+  source and fails with a schema diff if the generated artifact needs to be
+  committed. Do not edit the generated JSON schema by hand.
 
 - Do not default to full local workspace validation before every PR. Push/open
   the PR so GitHub Actions runs the broad build, typecheck, lint, test,

.github/workflows/ci.yml

@@ -166,6 +166,9 @@ jobs:
       - name: Install dependencies
         run: bun install --frozen-lockfile
 
+      - name: Check generated eval schema sync
+        run: bun run validate:eval-schema
+
       - name: Run tests
         run: bun run test
 

package.json

@@ -20,6 +20,8 @@
     "agentv:buildrun": "bun run build && bun apps/cli/dist/cli.js",
     "beads:check": "bun scripts/check-beads-context.ts",
     "debug:pi-sdk-tools": "bun scripts/debug-pi-sdk-tools.ts",
+    "generate:eval-schema": "bun packages/core/scripts/generate-eval-schema.ts",
+    "validate:eval-schema": "bun scripts/check-eval-schema.ts",
     "validate:promptfoo-export": "bun test scripts/export-promptfoo-config.test.ts",
     "validate:examples": "EVAL_CRITERIA=placeholder CUSTOM_SYSTEM_PROMPT=placeholder bun scripts/validate-example-evals.ts",
     "eval:baseline-check": "bun scripts/check-eval-baselines.ts",

packages/core/scripts/generate-eval-schema.ts

@@ -2,8 +2,8 @@
 import { spawn } from 'node:child_process';
 /**
  * Generates AgentV JSON schemas from Zod schemas.
+ * Run: bun run generate:eval-schema (from repo root)
  * Run: bun run generate:schema (from packages/core)
- * Or:  bun packages/core/scripts/generate-eval-schema.ts (from repo root)
  */
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';

packages/core/src/evaluation/validation/eval-file.schema.ts

@@ -3,8 +3,9 @@
  * Used to generate eval.schema.json for AI agent reference.
  *
  * IMPORTANT: This schema describes the YAML input format, not the parsed runtime types.
- * When adding new eval features, update this schema AND run `bun run generate:schema`
- * to regenerate eval.schema.json. The sync test will fail if they diverge.
+ * When adding new eval features, update this schema AND run
+ * `bun run validate:eval-schema` from the repo root. That command regenerates
+ * eval.schema.json and fails if the generated artifact needs to be committed.
  */
 import { z } from 'zod/v3';
 

scripts/check-eval-schema.ts

@@ -0,0 +1,56 @@
+#!/usr/bin/env bun
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const repoRoot = path.resolve(import.meta.dirname, '..');
+const schemaPath = 'skills-data/agentv-eval-writer/references/eval.schema.json';
+const schemaAbsolutePath = path.join(repoRoot, schemaPath);
+
+async function runOrExit(args: string[], options: { readonly cwd: string }): Promise<void> {
+  const proc = Bun.spawn(args, {
+    cwd: options.cwd,
+    stdout: 'inherit',
+    stderr: 'inherit',
+  });
+  const exitCode = await proc.exited;
+  if (exitCode !== 0) {
+    process.exit(exitCode);
+  }
+}
+
+async function printSchemaDiff(): Promise<void> {
+  await runOrExit(['git', 'diff', '--', schemaPath], { cwd: repoRoot });
+}
+
+async function readSchema(): Promise<string | null> {
+  try {
+    return await readFile(schemaAbsolutePath, 'utf8');
+  } catch (error) {
+    if (error instanceof Error && 'code' in error && error.code === 'ENOENT') {
+      return null;
+    }
+    throw error;
+  }
+}
+
+const before = await readSchema();
+await runOrExit([process.execPath, 'packages/core/scripts/generate-eval-schema.ts'], {
+  cwd: repoRoot,
+});
+const after = await readSchema();
+
+if (before === after) {
+  console.log(`OK: ${schemaPath} is generated from the Zod source`);
+  process.exit(0);
+}
+
+const message =
+  'Generated eval schema is out of sync. This command regenerated eval.schema.json from packages/core/src/evaluation/validation/eval-file.schema.ts; review and commit the generated diff instead of editing the JSON manually.';
+if (process.env.GITHUB_ACTIONS === 'true') {
+  console.error(`::error file=${schemaPath}::${message}`);
+}
+console.error(`[schema] ERROR: ${message}`);
+console.error('  Run locally: bun run validate:eval-schema');
+console.error(`  Generated file: ${schemaPath}`);
+await printSchemaDiff();
+process.exit(1);