feat: Prepare agent-eval for npm CLI usage

.github/workflows/experiment.yml

@@ -55,6 +55,7 @@ jobs:
             --concurrency "$CONCURRENCY" \
             --docker-image "$DOCKER_IMAGE" \
             --experiment "$EXPERIMENT_NAME" \
+            --experiments experiments/src \
             --artifacts artifacts
       - name: Upload experiment results
         if: ${{ always() }}

README.md

@@ -47,6 +47,33 @@ export const experiment: ExperimentConfig = {
 }
 ```
 
+When authoring experiments outside of this repository, import the helper from
+`@primer/agent-eval/config` to keep the experiment config typed:
+
+```ts
+import {createExperiment} from '@primer/agent-eval/config'
+
+export const experiment = createExperiment({
+  name: 'Example experiment',
+  description: 'Experiment config demonstrating different options',
+  models: ['gpt-5.5'],
+  evals: ['001-agent-uses-button-from-primer'],
+  treatments: [],
+})
+```
+
+Run local experiments with the `agent-eval` CLI:
+
+```sh
+COPILOT_GITHUB_TOKEN=... agent-eval --experiments ./experiments --experiment example
+```
+
+You can also provide a path directly to `--experiment`:
+
+```sh
+COPILOT_GITHUB_TOKEN=... agent-eval --experiment ./experiments/example.ts
+```
+
 The experiment config will specify:
 
 - A name for the experiment

packages/agent-eval/README.md

@@ -0,0 +1,41 @@
+# @primer/agent-eval
+
+Run Primer agent evaluation experiments from the command line or from Node.js.
+
+## CLI
+
+Install the package and run the `agent-eval` binary with a GitHub token:
+
+```sh
+COPILOT_GITHUB_TOKEN=... agent-eval --experiments ./experiments --experiment example
+```
+
+Use `--experiments` to load experiment files from a local directory. Experiment
+files may export an `experiment` named export or a default export. `--experiment`
+may also be a path to a local experiment file when you only want to run one
+experiment.
+
+## Experiment config authoring
+
+Use `createExperiment` from `@primer/agent-eval/config` to keep local experiment
+files typed:
+
+```ts
+import {createExperiment} from '@primer/agent-eval/config'
+
+export const experiment = createExperiment({
+  name: 'Example experiment',
+  description: 'Compare treatment behavior',
+  models: ['gpt-5.5'],
+  evals: ['001-agent-uses-button-from-primer'],
+  treatments: [],
+})
+```
+
+## Programmatic usage
+
+The package index exports the experiment loading and runner APIs:
+
+```ts
+import {loadExperimentConfigs, run} from '@primer/agent-eval'
+```

packages/agent-eval/package.json

@@ -1,13 +1,34 @@
 {
   "name": "@primer/agent-eval",
   "version": "0.0.0",
-  "private": true,
   "type": "module",
+  "bin": {
+    "agent-eval": "./dist/cli.js"
+  },
+  "types": "./dist/index.d.ts",
   "exports": {
-    "./config": "./src/config.ts"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    },
+    "./cli": {
+      "types": "./dist/cli.d.ts",
+      "default": "./dist/cli.js"
+    },
+    "./config": {
+      "types": "./dist/config.d.ts",
+      "default": "./src/config.ts"
+    }
+  },
+  "engines": {
+    "node": ">=24"
   },
+  "files": [
+    "dist",
+    "src/config.ts"
+  ],
   "scripts": {
-    "build": "rolldown -c",
+    "build": "rimraf dist && rolldown -c",
     "clean": "rimraf dist",
     "type-check": "tsc --noEmit",
     "watch": "rolldown -c -w"
@@ -21,16 +42,20 @@
     "url": "https://github.com/primer/agent-eval/issues"
   },
   "dependencies": {
+    "dockerode": "^5.0.0",
+    "tar-fs": "^3.1.2",
+    "tar-stream": "^3.2.0",
+    "zod": "^4.4.3"
+  },
+  "devDependencies": {
     "@primer/agent-evals": "workspace:*",
     "@primer/agent-experiment": "workspace:*",
     "@primer/agent-experiments": "workspace:*",
     "@primer/agent-sandbox": "workspace:*",
-    "zod": "^4.4.3"
-  },
-  "devDependencies": {
     "@types/node": "^26.0.1",
     "rimraf": "^6.1.3",
     "rolldown": "^1.1.0",
+    "rolldown-plugin-dts": "^0.25.2",
     "typescript": "^6.0.3"
   }
 }

packages/agent-eval/rolldown.config.ts

@@ -1,18 +1,34 @@
 import {defineConfig} from 'rolldown/config'
-import type {RolldownOptions} from 'rolldown'
-import packageJson from './package.json' with {type: 'json '}
+import {dts} from 'rolldown-plugin-dts'
+import packageJson from './package.json' with {type: 'json'}
 
-const dependencies = [...Object.keys(packageJson.dependencies), ...Object.keys(packageJson.devDependencies)]
+const bundledDependencies = new Set([
+  '@primer/agent-evals',
+  '@primer/agent-experiment',
+  '@primer/agent-experiments',
+  '@primer/agent-sandbox',
+])
+const dependencies = [...Object.keys(packageJson.dependencies), ...Object.keys(packageJson.devDependencies)].filter(
+  name => {
+    return !bundledDependencies.has(name)
+  },
+)
 const external = dependencies.map(name => {
   return new RegExp(`^${name}(/.*)?`)
 })
 
-const config: RolldownOptions = defineConfig({
-  input: 'src/cli.ts',
+const config = defineConfig({
+  input: {
+    cli: 'src/cli.ts',
+    config: 'src/config.ts',
+    index: 'src/index.ts',
+  },
   platform: 'node',
   external,
+  plugins: [dts({eager: true, sourcemap: true})],
   output: {
     dir: 'dist',
+    entryFileNames: '[name].js',
   },
 })
 

packages/agent-eval/src/cli.ts

@@ -1,12 +1,13 @@
+#!/usr/bin/env node
 import {randomUUID} from 'node:crypto'
 import {existsSync} from 'node:fs'
 import path from 'node:path'
 import fs from 'node:fs/promises'
 import {parseArgs} from 'node:util'
 import {get as getEval} from '@primer/agent-evals'
-import {list, find} from '@primer/agent-experiments'
 import {ControlTreatment, type ExperimentConfig, type Model} from '@primer/agent-experiment'
 import type {Treatment, TreatmentResult} from './treatment'
+import {listExperiments, loadExperimentConfigs} from './experiments.ts'
 import {resolveExperimentEval} from './eval'
 import {run} from './run'
 
@@ -34,6 +35,10 @@ const {values} = parseArgs({
       short: 'e',
       description: 'The file name of the experiment to run',
     },
+    experiments: {
+      type: 'string',
+      description: 'The directory containing local experiment files',
+    },
     'docker-image': {
       type: 'string',
       description:
@@ -49,24 +54,33 @@ const MAX_CONCURRENCY =
   Number.isFinite(parsedConcurrency) && Number.isInteger(parsedConcurrency) && parsedConcurrency >= 1
     ? parsedConcurrency
     : 1
-const experimentConfigs: Array<ExperimentConfig> = []
+let experimentConfigs: Array<ExperimentConfig> = []
 
 if (!existsSync(ARTIFACTS_DIR)) {
   await fs.mkdir(ARTIFACTS_DIR, {recursive: true})
 }
 
 if (values.experiment) {
-  const experiment = find(values.experiment)
-  if (experiment) {
-    experimentConfigs.push(experiment)
-  } else {
+  experimentConfigs = await loadExperimentConfigs({
+    experiment: values.experiment,
+    experimentsDirectory: values.experiments,
+  })
+  if (experimentConfigs.length === 0) {
     console.log('Experiments:')
     console.log(
-      list()
+      (
+        await listExperiments({
+          experimentsDirectory: values.experiments,
+        })
+      )
         .map(([name]) => name)
         .join('\n'),
     )
   }
+} else {
+  experimentConfigs = await loadExperimentConfigs({
+    experimentsDirectory: values.experiments,
+  })
 }
 
 function randomize<T>(input: Array<T>): Array<T> {

packages/agent-eval/src/config.ts

@@ -1,8 +1,25 @@
-import type {EvalConfig} from '@primer/agent-experiment'
+import type {
+  EvalConfig as ProjectEvalConfig,
+  ExperimentConfig as ProjectExperimentConfig,
+  Model as ProjectModel,
+  TreatmentConfig as ProjectTreatmentConfig,
+} from '@primer/agent-experiment'
+import type {McpServerConfig as ProjectMcpServerConfig, Sandbox as ProjectSandbox} from '@primer/agent-sandbox'
+
+type EvalConfig = ProjectEvalConfig
+type ExperimentConfig = ProjectExperimentConfig
+type McpServerConfig = ProjectMcpServerConfig
+type Model = ProjectModel
+type Sandbox = ProjectSandbox
+type TreatmentConfig = ProjectTreatmentConfig
 
 function defineConfig(config: EvalConfig) {
   return config
 }
 
-export type {EvalConfig}
-export {defineConfig}
+function createExperiment(config: ExperimentConfig) {
+  return config
+}
+
+export type {EvalConfig, ExperimentConfig, McpServerConfig, Model, Sandbox, TreatmentConfig}
+export {createExperiment, defineConfig}

packages/agent-eval/src/experiments.test.ts

@@ -0,0 +1,72 @@
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import {describe, expect, test} from 'vitest'
+import {findExperiment, listExperiments, loadExperimentConfigs} from './experiments.ts'
+
+async function createExperimentsDirectory() {
+  const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-experiments-'))
+  await fs.writeFile(
+    path.join(directory, 'example.mjs'),
+    `export const experiment = {
+      name: 'Example',
+      description: 'Example experiment',
+      models: ['gpt-5.5'],
+      evals: ['001-agent-uses-button-from-primer'],
+      treatments: []
+    }`,
+  )
+  await fs.writeFile(
+    path.join(directory, 'default-export.mjs'),
+    `export default {
+      name: 'Default export',
+      description: 'Default export experiment',
+      models: ['gpt-5.5'],
+      evals: ['001-agent-uses-button-from-primer'],
+      treatments: []
+    }`,
+  )
+  await fs.writeFile(path.join(directory, 'index.ts'), 'throw new Error("index should be ignored")')
+  return directory
+}
+
+describe('local experiment loading', () => {
+  test('lists experiments from a local directory', async () => {
+    const directory = await createExperimentsDirectory()
+
+    await expect(listExperiments({experimentsDirectory: directory})).resolves.toEqual([
+      ['default-export', expect.objectContaining({name: 'Default export'})],
+      ['example', expect.objectContaining({name: 'Example'})],
+    ])
+  })
+
+  test('finds a named experiment from a local directory', async () => {
+    const directory = await createExperimentsDirectory()
+
+    await expect(findExperiment('example', {experimentsDirectory: directory})).resolves.toEqual(
+      expect.objectContaining({name: 'Example'}),
+    )
+  })
+
+  test('finds an experiment from a local file path', async () => {
+    const directory = await createExperimentsDirectory()
+
+    await expect(findExperiment(path.join(directory, 'example.mjs'))).resolves.toEqual(
+      expect.objectContaining({name: 'Example'}),
+    )
+  })
+
+  test('loads an experiment from a local file path', async () => {
+    const directory = await createExperimentsDirectory()
+
+    await expect(loadExperimentConfigs({experiment: path.join(directory, 'example.mjs')})).resolves.toEqual([
+      expect.objectContaining({name: 'Example'}),
+    ])
+  })
+
+  test('loads all experiments when no experiment is specified', async () => {
+    const directory = await createExperimentsDirectory()
+
+    await expect(loadExperimentConfigs({experimentsDirectory: directory})).resolves.toHaveLength(2)
+  })
+})