fix(eval): reject authored direct input (#1646) #1070
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| concurrency: | |
| group: ci-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version-file: package.json | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Build packages | |
| run: bun run build | |
| - name: Stage build artifact | |
| env: | |
| COMMIT_SHA: ${{ github.sha }} | |
| RUNNER_ARCH_NAME: ${{ runner.arch }} | |
| RUNNER_OS_NAME: ${{ runner.os }} | |
| run: | | |
| set -euo pipefail | |
| artifact_dir="${RUNNER_TEMP}/agentv-build-artifact" | |
| rm -rf "$artifact_dir" | |
| mkdir -p "$artifact_dir" | |
| included_paths=( | |
| "packages/core/dist" | |
| "packages/sdk/dist" | |
| "apps/cli/dist" | |
| "apps/dashboard/dist" | |
| ) | |
| for path in "${included_paths[@]}"; do | |
| if [ ! -d "$path" ]; then | |
| echo "::error::Expected build output missing: $path" | |
| exit 1 | |
| fi | |
| mkdir -p "$artifact_dir/$(dirname "$path")" | |
| cp -R "$path" "$artifact_dir/$path" | |
| done | |
| forbidden_path=$(find "$artifact_dir" \ | |
| \( -path "*/node_modules/*" \ | |
| -o -path "*/.bun/*" \ | |
| -o -path "*/.turbo/*" \ | |
| -o -path "*/.cache/*" \ | |
| -o -name "*.tsbuildinfo" \) \ | |
| -print -quit) | |
| if [ -n "$forbidden_path" ]; then | |
| echo "::error::Build artifact contains a dependency, cache, or tsbuildinfo path: $forbidden_path" | |
| exit 1 | |
| fi | |
| MANIFEST_PATH="$artifact_dir/manifest.json" \ | |
| COMMIT_SHA="$COMMIT_SHA" \ | |
| RUNNER_ARCH_NAME="$RUNNER_ARCH_NAME" \ | |
| RUNNER_OS_NAME="$RUNNER_OS_NAME" \ | |
| bun -e ' | |
| import { createHash } from "node:crypto"; | |
| import { Buffer } from "node:buffer"; | |
| const lockfile = await Bun.file("bun.lock").arrayBuffer(); | |
| const rootPackageJson = await Bun.file("package.json").json(); | |
| const includedPaths = [ | |
| "packages/core/dist/**", | |
| "packages/sdk/dist/**", | |
| "apps/cli/dist/**", | |
| "apps/dashboard/dist/**", | |
| ]; | |
| const manifest = { | |
| commit_sha: process.env.COMMIT_SHA, | |
| bun_lock_sha256: createHash("sha256") | |
| .update(Buffer.from(lockfile)) | |
| .digest("hex"), | |
| runner_os: process.env.RUNNER_OS_NAME, | |
| runner_arch: process.env.RUNNER_ARCH_NAME, | |
| bun_version_source: "package.json#packageManager", | |
| bun_version_spec: rootPackageJson.packageManager ?? null, | |
| bun_version: Bun.version, | |
| built_at: new Date().toISOString(), | |
| included_paths: includedPaths, | |
| }; | |
| await Bun.write( | |
| process.env.MANIFEST_PATH, | |
| `${JSON.stringify(manifest, null, 2)}\n`, | |
| ); | |
| ' | |
| - name: Upload build artifact | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: agentv-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} | |
| path: ${{ runner.temp }}/agentv-build-artifact/ | |
| if-no-files-found: error | |
| retention-days: 10 | |
| typecheck: | |
| name: Typecheck | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version-file: package.json | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Typecheck packages | |
| run: bun run typecheck | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version-file: package.json | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run Biome | |
| run: bun run lint | |
| test: | |
| name: Test | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version-file: package.json | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Run tests | |
| run: bun run test | |
| links: | |
| name: Check Links | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Check relative markdown links | |
| uses: lycheeverse/lychee-action@v2 | |
| with: | |
| args: >- | |
| --offline | |
| --no-progress | |
| --glob-ignore-case | |
| --root-dir . | |
| "**/*.md" | |
| marketplace: | |
| name: Validate Marketplace | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version-file: package.json | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Validate marketplace.json (schema + sync) | |
| run: bun scripts/marketplace/validate-marketplace.ts | |
| - name: Check marketplace sorted | |
| run: bun scripts/marketplace/check-sorted.ts | |
| - name: Validate frontmatter | |
| run: bun scripts/marketplace/validate-frontmatter.ts | |
| evals: | |
| name: Validate Evals | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: build | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Setup Bun | |
| uses: oven-sh/setup-bun@v2 | |
| with: | |
| bun-version-file: package.json | |
| - name: Install dependencies | |
| run: bun install --frozen-lockfile | |
| - name: Download build artifact | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: agentv-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} | |
| path: ${{ runner.temp }}/agentv-build-artifact | |
| - name: Restore build artifact | |
| env: | |
| ARTIFACT_DIR: ${{ runner.temp }}/agentv-build-artifact | |
| EXPECTED_COMMIT_SHA: ${{ github.sha }} | |
| EXPECTED_RUNNER_ARCH: ${{ runner.arch }} | |
| EXPECTED_RUNNER_OS: ${{ runner.os }} | |
| run: | | |
| set -euo pipefail | |
| bun -e ' | |
| import { createHash } from "node:crypto"; | |
| import { Buffer } from "node:buffer"; | |
| import { cpSync, existsSync, mkdirSync, rmSync } from "node:fs"; | |
| import path from "node:path"; | |
| const artifactDir = process.env.ARTIFACT_DIR; | |
| if (!artifactDir) { | |
| throw new Error("ARTIFACT_DIR is required"); | |
| } | |
| const manifestPath = path.join(artifactDir, "manifest.json"); | |
| if (!existsSync(manifestPath)) { | |
| throw new Error(`Build artifact manifest is missing: ${manifestPath}`); | |
| } | |
| const manifest = await Bun.file(manifestPath).json(); | |
| const rootPackageJson = await Bun.file("package.json").json(); | |
| const lockfile = await Bun.file("bun.lock").arrayBuffer(); | |
| const lockHash = createHash("sha256") | |
| .update(Buffer.from(lockfile)) | |
| .digest("hex"); | |
| const expected = { | |
| commit_sha: process.env.EXPECTED_COMMIT_SHA, | |
| bun_lock_sha256: lockHash, | |
| runner_os: process.env.EXPECTED_RUNNER_OS, | |
| runner_arch: process.env.EXPECTED_RUNNER_ARCH, | |
| bun_version_spec: rootPackageJson.packageManager ?? null, | |
| bun_version: Bun.version, | |
| }; | |
| for (const [key, value] of Object.entries(expected)) { | |
| if (manifest[key] !== value) { | |
| throw new Error( | |
| `Build artifact manifest mismatch for ${key}: expected ${value}, got ${manifest[key]}`, | |
| ); | |
| } | |
| } | |
| const requiredPaths = [ | |
| "packages/core/dist", | |
| "packages/sdk/dist", | |
| "apps/cli/dist", | |
| "apps/dashboard/dist", | |
| ]; | |
| const includedPaths = new Set(manifest.included_paths ?? []); | |
| for (const relativePath of requiredPaths) { | |
| if (!includedPaths.has(`${relativePath}/**`)) { | |
| throw new Error(`Build artifact manifest does not include ${relativePath}/**`); | |
| } | |
| const source = path.join(artifactDir, relativePath); | |
| if (!existsSync(source)) { | |
| throw new Error(`Build artifact path is missing: ${source}`); | |
| } | |
| rmSync(relativePath, { recursive: true, force: true }); | |
| mkdirSync(path.dirname(relativePath), { recursive: true }); | |
| cpSync(source, relativePath, { recursive: true }); | |
| } | |
| ' | |
| - name: Validate eval schemas | |
| run: bun apps/cli/dist/cli.js validate 'examples/features/**/evals/**/suite.yaml' 'examples/features/**/evals/**/*.eval.yaml' 'examples/features/**/*.EVAL.yaml' |