From 58920143bae7ceecc44595aa72c340394c92c857 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:11:40 +0000 Subject: [PATCH 1/4] Add npm CLI package support --- .github/workflows/experiment.yml | 1 + README.md | 21 +++++ packages/agent-eval/package.json | 24 ++++-- packages/agent-eval/rolldown.config.ts | 13 ++- packages/agent-eval/src/cli.ts | 28 +++++-- packages/agent-eval/src/config.ts | 65 ++++++++++++++- packages/agent-eval/src/experiments.test.ts | 56 +++++++++++++ packages/agent-eval/src/experiments.ts | 90 +++++++++++++++++++++ packages/agent-eval/tsconfig.build.json | 12 +++ pnpm-lock.yaml | 4 - 10 files changed, 292 insertions(+), 22 deletions(-) create mode 100644 packages/agent-eval/src/experiments.test.ts create mode 100644 packages/agent-eval/src/experiments.ts create mode 100644 packages/agent-eval/tsconfig.build.json diff --git a/.github/workflows/experiment.yml b/.github/workflows/experiment.yml index 0600046..93ccca9 100644 --- a/.github/workflows/experiment.yml +++ b/.github/workflows/experiment.yml @@ -46,6 +46,7 @@ jobs: node packages/agent-eval/dist/cli.js \ --concurrency "$CONCURRENCY" \ --experiment "$EXPERIMENT_NAME" \ + --experiments experiments/src \ --artifacts artifacts - name: Upload experiment results if: ${{ always() }} diff --git a/README.md b/README.md index e2f6d6d..c9a1b9f 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,27 @@ export const experiment: ExperimentConfig = { } ``` +When authoring experiments outside of this repository, import the helper from +`@primer/agent-eval/config` to keep the experiment config typed: + +```ts +import {defineExperimentConfig} from '@primer/agent-eval/config' + +export const experiment = defineExperimentConfig({ + name: 'Example experiment', + description: 'Experiment config demonstrating different options', + models: ['gpt-5.5'], + evals: ['001-agent-uses-button-from-primer'], + treatments: [], +}) +``` + +Run local experiments with the `agent-eval` CLI: + +```sh +COPILOT_GITHUB_TOKEN=... agent-eval --experiments ./experiments --experiment example +``` + The experiment config will specify: - A name for the experiment diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index af643a1..7273196 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -1,13 +1,25 @@ { "name": "@primer/agent-eval", "version": "0.0.0", - "private": true, "type": "module", + "bin": { + "agent-eval": "./dist/cli.js" + }, "exports": { - "./config": "./src/config.ts" + ".": { + "types": "./dist/config.d.ts", + "import": "./dist/config.js" + }, + "./config": { + "types": "./dist/config.d.ts", + "import": "./dist/config.js" + } }, + "files": [ + "dist" + ], "scripts": { - "build": "rolldown -c", + "build": "rimraf dist && rolldown -c && tsc -p tsconfig.build.json", "clean": "rimraf dist", "type-check": "tsc --noEmit", "watch": "rolldown -c -w" @@ -21,13 +33,13 @@ "url": "https://github.com/primer/agent-eval/issues" }, "dependencies": { + "zod": "^4.4.3" + }, + "devDependencies": { "@primer/agent-evals": "workspace:*", "@primer/agent-experiment": "workspace:*", "@primer/agent-experiments": "workspace:*", "@primer/agent-sandbox": "workspace:*", - "zod": "^4.4.3" - }, - "devDependencies": { "@types/node": "^24.12.4", "rimraf": "^6.1.3", "rolldown": "^1.0.1", diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts index a46d984..d9e6e15 100644 --- a/packages/agent-eval/rolldown.config.ts +++ b/packages/agent-eval/rolldown.config.ts @@ -1,16 +1,23 @@ import {defineConfig} from 'rolldown/config' -import packageJson from './package.json' with {type: 'json '} +import packageJson from './package.json' with {type: 'json'} -const dependencies = [...Object.keys(packageJson.dependencies), ...Object.keys(packageJson.devDependencies)] +const bundledDependencies = new Set(['@primer/agent-evals', '@primer/agent-experiment', '@primer/agent-experiments', '@primer/agent-sandbox']) +const dependencies = [...Object.keys(packageJson.dependencies), ...Object.keys(packageJson.devDependencies)].filter(name => { + return !bundledDependencies.has(name) +}) const external = dependencies.map(name => { return new RegExp(`^${name}(/.*)?`) }) export default defineConfig({ - input: 'src/cli.ts', + input: { + cli: 'src/cli.ts', + config: 'src/config.ts', + }, platform: 'node', external, output: { dir: 'dist', + entryFileNames: '[name].js', }, }) diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index 07ce106..b3556b6 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -1,12 +1,13 @@ +#!/usr/bin/env node import {randomUUID} from 'node:crypto' import {existsSync} from 'node:fs' import path from 'node:path' import fs from 'node:fs/promises' import {parseArgs} from 'node:util' import {get as getEval} from '@primer/agent-evals' -import {list, find} from '@primer/agent-experiments' import {ControlTreatment, type ExperimentConfig, type Model} from '@primer/agent-experiment' import type {Treatment, TreatmentResult} from './treatment' +import {listExperiments, loadExperimentConfigs} from './experiments.ts' import {run} from './run' const COPILOT_GITHUB_TOKEN = process.env.COPILOT_GITHUB_TOKEN @@ -33,6 +34,10 @@ const {values} = parseArgs({ short: 'e', description: 'The file name of the experiment to run', }, + experiments: { + type: 'string', + description: 'The directory containing local experiment files', + }, }, }) @@ -42,24 +47,33 @@ const MAX_CONCURRENCY = Number.isFinite(parsedConcurrency) && Number.isInteger(parsedConcurrency) && parsedConcurrency >= 1 ? parsedConcurrency : 1 -const experimentConfigs: Array = [] +let experimentConfigs: Array = [] if (!existsSync(ARTIFACTS_DIR)) { await fs.mkdir(ARTIFACTS_DIR, {recursive: true}) } if (values.experiment) { - const experiment = find(values.experiment) - if (experiment) { - experimentConfigs.push(experiment) - } else { + experimentConfigs = await loadExperimentConfigs({ + experiment: values.experiment, + experimentsDirectory: values.experiments, + }) + if (experimentConfigs.length === 0) { console.log('Experiments:') console.log( - list() + ( + await listExperiments({ + experimentsDirectory: values.experiments, + }) + ) .map(([name]) => name) .join('\n'), ) } +} else { + experimentConfigs = await loadExperimentConfigs({ + experimentsDirectory: values.experiments, + }) } function randomize(input: Array): Array { diff --git a/packages/agent-eval/src/config.ts b/packages/agent-eval/src/config.ts index c32f666..684b227 100644 --- a/packages/agent-eval/src/config.ts +++ b/packages/agent-eval/src/config.ts @@ -2,9 +2,70 @@ type EvalConfig = { prompt: string } +type Model = + | 'claude-haiku-4.5' + | 'claude-opus-4.6' + | 'claude-opus-4.7' + | 'claude-sonnet-4.5' + | 'claude-sonnet-4.6' + | 'gpt-5.4' + | 'gpt-5.4-mini' + | 'gpt-5.5' + +type CommandResult = { + stdout: string + stderr: string + exitCode: number +} + +type McpServerConfig = { + command: string + type: 'local' + args?: Array + env?: Record + tools?: Array +} + +type Sandbox = { + addAgentInstruction(text: string): Promise + addAgentSkill(name: string, description: string, contents: string): Promise + addMcpServer(name: string, config: McpServerConfig): Promise + copy(sourcePath: string, destinationPath: string, options?: {exclude?: Array}): Promise + download( + containerFilePath: string, + hostDestinationPath: string, + options?: {ignore?: (name: string) => boolean}, + ): Promise + exists(filepath: string): Promise + readFile(filepath: string): Promise + runCommand( + command: string, + args?: Array, + options?: {env?: Record; user?: string; allowNonZeroExitCode?: boolean}, + ): Promise + writeFile(filepath: string, contents: string): Promise +} + +type ExperimentConfig = { + name: string + description: string + models: Array + evals: Array + treatments: Array +} + +type TreatmentConfig = { + name: string + setup?: ({sandbox}: {sandbox: Sandbox}) => Promise +} + function defineConfig(config: EvalConfig) { return config } -export type {EvalConfig} -export {defineConfig} +function defineExperimentConfig(config: ExperimentConfig) { + return config +} + +export type {EvalConfig, ExperimentConfig, McpServerConfig, Model, Sandbox, TreatmentConfig} +export {defineConfig, defineExperimentConfig} diff --git a/packages/agent-eval/src/experiments.test.ts b/packages/agent-eval/src/experiments.test.ts new file mode 100644 index 0000000..be5321d --- /dev/null +++ b/packages/agent-eval/src/experiments.test.ts @@ -0,0 +1,56 @@ +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import {describe, expect, test} from 'vitest' +import {findExperiment, listExperiments, loadExperimentConfigs} from './experiments.ts' + +async function createExperimentsDirectory() { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'agent-eval-experiments-')) + await fs.writeFile( + path.join(directory, 'example.mjs'), + `export const experiment = { + name: 'Example', + description: 'Example experiment', + models: ['gpt-5.5'], + evals: ['001-agent-uses-button-from-primer'], + treatments: [] + }`, + ) + await fs.writeFile( + path.join(directory, 'default-export.mjs'), + `export default { + name: 'Default export', + description: 'Default export experiment', + models: ['gpt-5.5'], + evals: ['001-agent-uses-button-from-primer'], + treatments: [] + }`, + ) + await fs.writeFile(path.join(directory, 'index.ts'), 'throw new Error("index should be ignored")') + return directory +} + +describe('local experiment loading', () => { + test('lists experiments from a local directory', async () => { + const directory = await createExperimentsDirectory() + + await expect(listExperiments({experimentsDirectory: directory})).resolves.toEqual([ + ['default-export', expect.objectContaining({name: 'Default export'})], + ['example', expect.objectContaining({name: 'Example'})], + ]) + }) + + test('finds a named experiment from a local directory', async () => { + const directory = await createExperimentsDirectory() + + await expect(findExperiment('example', {experimentsDirectory: directory})).resolves.toEqual( + expect.objectContaining({name: 'Example'}), + ) + }) + + test('loads all experiments when no experiment is specified', async () => { + const directory = await createExperimentsDirectory() + + await expect(loadExperimentConfigs({experimentsDirectory: directory})).resolves.toHaveLength(2) + }) +}) diff --git a/packages/agent-eval/src/experiments.ts b/packages/agent-eval/src/experiments.ts new file mode 100644 index 0000000..04a957b --- /dev/null +++ b/packages/agent-eval/src/experiments.ts @@ -0,0 +1,90 @@ +import {existsSync} from 'node:fs' +import fs from 'node:fs/promises' +import path from 'node:path' +import {pathToFileURL} from 'node:url' +import {find as findPackagedExperiment, list as listPackagedExperiments} from '@primer/agent-experiments' +import type {ExperimentConfig} from '@primer/agent-experiment' + +type ExperimentModule = { + default?: ExperimentConfig + experiment?: ExperimentConfig +} + +type ExperimentSourceOptions = { + experimentsDirectory?: string +} + +type LoadExperimentOptions = ExperimentSourceOptions & { + experiment?: string +} + +const EXPERIMENT_FILE_EXTENSIONS = new Set(['.cjs', '.js', '.mjs', '.ts']) + +function getExperimentId(filename: string): string { + return filename.replace(/\.(?:cjs|js|mjs|ts)$/, '') +} + +function isExperimentFile(filename: string): boolean { + if (filename.endsWith('.d.ts')) { + return false + } + + return filename !== 'index.ts' && EXPERIMENT_FILE_EXTENSIONS.has(path.extname(filename)) +} + +async function getLocalExperimentEntries(experimentsDirectory: string): Promise> { + const directory = path.resolve(experimentsDirectory) + if (!existsSync(directory)) { + throw new Error(`Experiments directory does not exist: ${directory}`) + } + + const stats = await fs.stat(directory) + if (!stats.isDirectory()) { + throw new Error(`Experiments path is not a directory: ${directory}`) + } + + const filenames = (await fs.readdir(directory)).toSorted() + return Promise.all( + filenames.filter(isExperimentFile).map(async filename => { + const filepath = path.join(directory, filename) + const mod = (await import(pathToFileURL(filepath).href)) as ExperimentModule + const experiment = mod.experiment ?? mod.default + if (!experiment) { + throw new Error(`Experiment file must export "experiment" or a default export: ${filepath}`) + } + + return [getExperimentId(filename), experiment] + }), + ) +} + +async function listExperiments(options: ExperimentSourceOptions = {}): Promise> { + if (options.experimentsDirectory) { + return getLocalExperimentEntries(options.experimentsDirectory) + } + + return listPackagedExperiments() +} + +async function findExperiment( + id: string, + options: ExperimentSourceOptions = {}, +): Promise { + if (options.experimentsDirectory) { + const experiments = await getLocalExperimentEntries(options.experimentsDirectory) + return experiments.find(([name]) => name === id)?.[1] + } + + return findPackagedExperiment(id) +} + +async function loadExperimentConfigs(options: LoadExperimentOptions = {}): Promise> { + if (options.experiment) { + const experiment = await findExperiment(options.experiment, options) + return experiment ? [experiment] : [] + } + + return (await listExperiments(options)).map(([, experiment]) => experiment) +} + +export {findExperiment, listExperiments, loadExperimentConfigs} diff --git a/packages/agent-eval/tsconfig.build.json b/packages/agent-eval/tsconfig.build.json new file mode 100644 index 0000000..228b316 --- /dev/null +++ b/packages/agent-eval/tsconfig.build.json @@ -0,0 +1,12 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "emitDeclarationOnly": true, + "noEmit": false, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9dc6a33..c146fb5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2414,10 +2414,6 @@ packages: resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} engines: {node: '>=12.0.0'} - tinyglobby@0.2.17: - resolution: {integrity: sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==} - engines: {node: '>=12.0.0'} - tinyrainbow@3.1.0: resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} engines: {node: '>=14.0.0'} From 37080cc3fe447147e74e2352d0e93105cdc930d6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 11 Jun 2026 16:16:10 +0000 Subject: [PATCH 2/4] Finalize npm package metadata --- packages/agent-eval/package.json | 15 +++++++++------ packages/agent-eval/rolldown.config.ts | 15 +++++++++++---- packages/agent-eval/src/cli.ts | 2 +- pnpm-lock.yaml | 22 +++++++++++++--------- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index 7273196..85fb826 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -6,17 +6,17 @@ "agent-eval": "./dist/cli.js" }, "exports": { - ".": { - "types": "./dist/config.d.ts", - "import": "./dist/config.js" - }, "./config": { "types": "./dist/config.d.ts", - "import": "./dist/config.js" + "default": "./src/config.ts" } }, + "engines": { + "node": ">=24" + }, "files": [ - "dist" + "dist", + "src/config.ts" ], "scripts": { "build": "rimraf dist && rolldown -c && tsc -p tsconfig.build.json", @@ -33,6 +33,9 @@ "url": "https://github.com/primer/agent-eval/issues" }, "dependencies": { + "dockerode": "^5.0.0", + "tar-fs": "^3.1.2", + "tar-stream": "^3.2.0", "zod": "^4.4.3" }, "devDependencies": { diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts index d9e6e15..fcc0245 100644 --- a/packages/agent-eval/rolldown.config.ts +++ b/packages/agent-eval/rolldown.config.ts @@ -1,10 +1,17 @@ import {defineConfig} from 'rolldown/config' import packageJson from './package.json' with {type: 'json'} -const bundledDependencies = new Set(['@primer/agent-evals', '@primer/agent-experiment', '@primer/agent-experiments', '@primer/agent-sandbox']) -const dependencies = [...Object.keys(packageJson.dependencies), ...Object.keys(packageJson.devDependencies)].filter(name => { - return !bundledDependencies.has(name) -}) +const bundledDependencies = new Set([ + '@primer/agent-evals', + '@primer/agent-experiment', + '@primer/agent-experiments', + '@primer/agent-sandbox', +]) +const dependencies = [...Object.keys(packageJson.dependencies), ...Object.keys(packageJson.devDependencies)].filter( + name => { + return !bundledDependencies.has(name) + }, +) const external = dependencies.map(name => { return new RegExp(`^${name}(/.*)?`) }) diff --git a/packages/agent-eval/src/cli.ts b/packages/agent-eval/src/cli.ts index b3556b6..f89819b 100644 --- a/packages/agent-eval/src/cli.ts +++ b/packages/agent-eval/src/cli.ts @@ -63,7 +63,7 @@ if (values.experiment) { console.log( ( await listExperiments({ - experimentsDirectory: values.experiments, + experimentsDirectory: values.experiments, }) ) .map(([name]) => name) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c146fb5..c194b20 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -205,6 +205,19 @@ importers: packages/agent-eval: dependencies: + dockerode: + specifier: ^5.0.0 + version: 5.0.0 + tar-fs: + specifier: ^3.1.2 + version: 3.1.2 + tar-stream: + specifier: ^3.2.0 + version: 3.2.0 + zod: + specifier: ^4.4.3 + version: 4.4.3 + devDependencies: '@primer/agent-evals': specifier: workspace:* version: link:../evals @@ -217,10 +230,6 @@ importers: '@primer/agent-sandbox': specifier: workspace:* version: link:../sandbox - zod: - specifier: ^4.4.3 - version: 4.4.3 - devDependencies: '@types/node': specifier: ^24.12.4 version: 24.12.4 @@ -4962,11 +4971,6 @@ snapshots: tinyexec@1.2.4: {} - tinyglobby@0.2.17: - dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - tinyglobby@0.2.17: dependencies: fdir: 6.5.0(picomatch@4.0.4) From 54c4dfac833272c1c7a1f93d1c41200bf22c3844 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Jun 2026 15:14:02 +0000 Subject: [PATCH 3/4] Address agent-eval package review feedback --- README.md | 4 +- packages/agent-eval/README.md | 39 ++++++ packages/agent-eval/package.json | 12 +- packages/agent-eval/rolldown.config.ts | 3 + packages/agent-eval/src/config.ts | 72 ++--------- packages/agent-eval/src/index.ts | 4 + packages/sandbox/src/index.ts | 1 + pnpm-lock.yaml | 169 +++++++++++++++++++++++++ 8 files changed, 243 insertions(+), 61 deletions(-) create mode 100644 packages/agent-eval/README.md create mode 100644 packages/agent-eval/src/index.ts diff --git a/README.md b/README.md index c9a1b9f..e384f4e 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,9 @@ When authoring experiments outside of this repository, import the helper from `@primer/agent-eval/config` to keep the experiment config typed: ```ts -import {defineExperimentConfig} from '@primer/agent-eval/config' +import {createExperiment} from '@primer/agent-eval/config' -export const experiment = defineExperimentConfig({ +export const experiment = createExperiment({ name: 'Example experiment', description: 'Experiment config demonstrating different options', models: ['gpt-5.5'], diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md new file mode 100644 index 0000000..340b1a9 --- /dev/null +++ b/packages/agent-eval/README.md @@ -0,0 +1,39 @@ +# @primer/agent-eval + +Run Primer agent evaluation experiments from the command line or from Node.js. + +## CLI + +Install the package and run the `agent-eval` binary with a GitHub token: + +```sh +COPILOT_GITHUB_TOKEN=... agent-eval --experiments ./experiments --experiment example +``` + +Use `--experiments` to load experiment files from a local directory. Experiment +files may export an `experiment` named export or a default export. + +## Experiment config authoring + +Use `createExperiment` from `@primer/agent-eval/config` to keep local experiment +files typed: + +```ts +import {createExperiment} from '@primer/agent-eval/config' + +export const experiment = createExperiment({ + name: 'Example experiment', + description: 'Compare treatment behavior', + models: ['gpt-5.5'], + evals: ['001-agent-uses-button-from-primer'], + treatments: [], +}) +``` + +## Programmatic usage + +The package index exports the experiment loading and runner APIs: + +```ts +import {loadExperimentConfigs, run} from '@primer/agent-eval' +``` diff --git a/packages/agent-eval/package.json b/packages/agent-eval/package.json index 85fb826..1d35a5a 100644 --- a/packages/agent-eval/package.json +++ b/packages/agent-eval/package.json @@ -5,7 +5,16 @@ "bin": { "agent-eval": "./dist/cli.js" }, + "types": "./dist/index.d.ts", "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./cli": { + "types": "./dist/cli.d.ts", + "default": "./dist/cli.js" + }, "./config": { "types": "./dist/config.d.ts", "default": "./src/config.ts" @@ -19,7 +28,7 @@ "src/config.ts" ], "scripts": { - "build": "rimraf dist && rolldown -c && tsc -p tsconfig.build.json", + "build": "rimraf dist && rolldown -c", "clean": "rimraf dist", "type-check": "tsc --noEmit", "watch": "rolldown -c -w" @@ -46,6 +55,7 @@ "@types/node": "^24.12.4", "rimraf": "^6.1.3", "rolldown": "^1.0.1", + "rolldown-plugin-dts": "^0.25.2", "typescript": "^6.0.3" } } diff --git a/packages/agent-eval/rolldown.config.ts b/packages/agent-eval/rolldown.config.ts index fcc0245..b6e0d27 100644 --- a/packages/agent-eval/rolldown.config.ts +++ b/packages/agent-eval/rolldown.config.ts @@ -1,4 +1,5 @@ import {defineConfig} from 'rolldown/config' +import {dts} from 'rolldown-plugin-dts' import packageJson from './package.json' with {type: 'json'} const bundledDependencies = new Set([ @@ -20,9 +21,11 @@ export default defineConfig({ input: { cli: 'src/cli.ts', config: 'src/config.ts', + index: 'src/index.ts', }, platform: 'node', external, + plugins: [dts({eager: true, sourcemap: true})], output: { dir: 'dist', entryFileNames: '[name].js', diff --git a/packages/agent-eval/src/config.ts b/packages/agent-eval/src/config.ts index 684b227..45b6c49 100644 --- a/packages/agent-eval/src/config.ts +++ b/packages/agent-eval/src/config.ts @@ -1,71 +1,27 @@ +import type { + ExperimentConfig as ProjectExperimentConfig, + Model as ProjectModel, + TreatmentConfig as ProjectTreatmentConfig, +} from '@primer/agent-experiment' +import type {McpServerConfig as ProjectMcpServerConfig, Sandbox as ProjectSandbox} from '@primer/agent-sandbox' + type EvalConfig = { prompt: string } -type Model = - | 'claude-haiku-4.5' - | 'claude-opus-4.6' - | 'claude-opus-4.7' - | 'claude-sonnet-4.5' - | 'claude-sonnet-4.6' - | 'gpt-5.4' - | 'gpt-5.4-mini' - | 'gpt-5.5' - -type CommandResult = { - stdout: string - stderr: string - exitCode: number -} - -type McpServerConfig = { - command: string - type: 'local' - args?: Array - env?: Record - tools?: Array -} - -type Sandbox = { - addAgentInstruction(text: string): Promise - addAgentSkill(name: string, description: string, contents: string): Promise - addMcpServer(name: string, config: McpServerConfig): Promise - copy(sourcePath: string, destinationPath: string, options?: {exclude?: Array}): Promise - download( - containerFilePath: string, - hostDestinationPath: string, - options?: {ignore?: (name: string) => boolean}, - ): Promise - exists(filepath: string): Promise - readFile(filepath: string): Promise - runCommand( - command: string, - args?: Array, - options?: {env?: Record; user?: string; allowNonZeroExitCode?: boolean}, - ): Promise - writeFile(filepath: string, contents: string): Promise -} - -type ExperimentConfig = { - name: string - description: string - models: Array - evals: Array - treatments: Array -} - -type TreatmentConfig = { - name: string - setup?: ({sandbox}: {sandbox: Sandbox}) => Promise -} +type ExperimentConfig = ProjectExperimentConfig +type McpServerConfig = ProjectMcpServerConfig +type Model = ProjectModel +type Sandbox = ProjectSandbox +type TreatmentConfig = ProjectTreatmentConfig function defineConfig(config: EvalConfig) { return config } -function defineExperimentConfig(config: ExperimentConfig) { +function createExperiment(config: ExperimentConfig) { return config } export type {EvalConfig, ExperimentConfig, McpServerConfig, Model, Sandbox, TreatmentConfig} -export {defineConfig, defineExperimentConfig} +export {createExperiment, defineConfig} diff --git a/packages/agent-eval/src/index.ts b/packages/agent-eval/src/index.ts new file mode 100644 index 0000000..48b4f5a --- /dev/null +++ b/packages/agent-eval/src/index.ts @@ -0,0 +1,4 @@ +export {findExperiment, listExperiments, loadExperimentConfigs} from './experiments.ts' +export {run} from './run' +export type {Treatment, TreatmentResult} from './treatment' +export {createExperiment, defineConfig} from './config.ts' diff --git a/packages/sandbox/src/index.ts b/packages/sandbox/src/index.ts index e176da8..042020b 100644 --- a/packages/sandbox/src/index.ts +++ b/packages/sandbox/src/index.ts @@ -572,3 +572,4 @@ function captureStream(destination: NodeJS.WritableStream): {stream: Writable; r } export {CONTAINER_WORKDIR, COPILOT_DIR, SKILLS_DIR, AGENTS_DIR, NODE_USER, Sandbox} +export type {McpServerConfig} from './mcp-config.ts' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c194b20..26ba8f5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -239,6 +239,9 @@ importers: rolldown: specifier: ^1.0.1 version: 1.0.1 + rolldown-plugin-dts: + specifier: ^0.25.2 + version: 0.25.2(rolldown@1.0.1)(typescript@6.0.3) typescript: specifier: ^6.0.3 version: 6.0.3 @@ -306,10 +309,40 @@ importers: packages: + '@babel/generator@8.0.0-rc.6': + resolution: {integrity: sha512-6mIzgVK8DgEzvIapoQwhXTMnnkuE4STQmVv9H03i/tZ2ml8oev3TRvZJgTenK2Bsq0YWNtzOrFdTyNzCMFtjJQ==} + engines: {node: ^22.18.0 || >=24.11.0} + + '@babel/helper-string-parser@8.0.0': + resolution: {integrity: sha512-6mJgmFFFIIO82vvoLt9XtRC7/TkzXfts1t/SpRX4IHSzMgqoPYCWesVu1udUPUWioAE/2fcG6WuI8zrkE1gwrg==} + engines: {node: ^22.18.0 || >=24.11.0} + + '@babel/helper-validator-identifier@8.0.0-rc.6': + resolution: {integrity: sha512-nVJ+1JcCgntv8d78rRo++o2wuODT0Irknx2BF8Np4Ft2CRgjLqIs4qzSZ8b66yGbBdMWGmZBO9WEZv1hhNiSpg==} + engines: {node: ^22.18.0 || >=24.11.0} + + '@babel/helper-validator-identifier@8.0.2': + resolution: {integrity: sha512-9Fr9QeyCAyi1BR1jKZ6uYQ24EIhQUx5ReHfQU7drOE+TPOb+w11/dsqLkMOT2U29OdCT71XajrOT8xDc1C7orA==} + engines: {node: ^22.18.0 || >=24.11.0} + + '@babel/parser@8.0.0': + resolution: {integrity: sha512-aLxAE+imI9bCcyaPrUDjBv3uSkWieifjLe0kuFOZF0zli0L6GCsTmsePnTr55adbIAgYz2zhN1vnFimCBUYcRQ==} + engines: {node: ^22.18.0 || >=24.11.0} + hasBin: true + + '@babel/parser@8.0.0-rc.6': + resolution: {integrity: sha512-rOS8IpdO7mQELkTPlCsTgPejO0bFuZdEDCGQJouYbYf9e1FLTym7Fei2pEjq8q7MWbX0ravcd7QQYKs1TxOuog==} + engines: {node: ^22.18.0 || >=24.11.0} + hasBin: true + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} + '@babel/types@8.0.0': + resolution: {integrity: sha512-K8ponJDxBwDHigkeFqaqT5wLGl4bTlwMafR8k7b5CPxr6Ww+UG9ls8Yx6Tcpboxu97eeGVEEyKcHmEyOwN1vSw==} + engines: {node: ^22.18.0 || >=24.11.0} + '@balena/dockerignore@1.0.2': resolution: {integrity: sha512-wMue2Sy4GAVTk6Ic4tJVcnfdau+gx2EnG7S+uAEe+TWJFqE4YoWN4/H8MSLj4eYJKxGg26lZwboEniNiNwZQ6Q==} @@ -571,9 +604,19 @@ packages: cpu: [x64] os: [win32] + '@jridgewell/gen-mapping@0.3.13': + resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} + + '@jridgewell/resolve-uri@3.1.2': + resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} + engines: {node: '>=6.0.0'} + '@jridgewell/sourcemap-codec@1.5.5': resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} + '@jridgewell/trace-mapping@0.3.31': + resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@js-sdsl/ordered-map@4.4.2': resolution: {integrity: sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==} @@ -881,6 +924,9 @@ packages: '@types/estree@1.0.9': resolution: {integrity: sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==} + '@types/jsesc@2.5.1': + resolution: {integrity: sha512-9VN+6yxLOPLOav+7PwjZbxiID2bVaeq0ED4qSQmdQTdjnXJSaCVKTR58t15oqH1H5t8Ng2ZX1SabJVoN9Q34bw==} + '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} @@ -1116,6 +1162,10 @@ packages: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + ast-kit@3.0.0: + resolution: {integrity: sha512-8OG92q3R35qjC/4i6BLBMg8IB+fClWu/1PEwg2Z9Rn+BuNaiEgJzpzn+pxWOdHJWDCAwu2JP0wCDTozAM4QirQ==} + engines: {node: ^22.18.0 || >=24.11.0} + ast-types-flow@0.0.8: resolution: {integrity: sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==} @@ -1202,6 +1252,9 @@ packages: bcrypt-pbkdf@1.0.2: resolution: {integrity: sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==} + birpc@4.0.0: + resolution: {integrity: sha512-LShSxJP0KTmd101b6DRyGBj57LZxSDYWKitQNW/mi8GRMvZb078Uf9+pveax1DrVL89vm7mWe+TovdI/UDOuPw==} + bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -1355,6 +1408,15 @@ packages: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} + dts-resolver@3.0.0: + resolution: {integrity: sha512-1T1f+z+4tl9XD+m+0HBgWoL/nm0bOIffyWaUuUSBlFg/86IWvfx+wjNaO/ybU0AJzG9/Mi5hBUgGV6zCmWEN7Q==} + engines: {node: ^22.18.0 || >=24.0.0} + peerDependencies: + oxc-resolver: '>=11.0.0' + peerDependenciesMeta: + oxc-resolver: + optional: true + dunder-proto@1.0.1: resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} engines: {node: '>= 0.4'} @@ -1649,6 +1711,10 @@ packages: resolution: {integrity: sha512-w9UMqWwJxHNOvoNzSJ2oPF5wvYcvP7jUvYzhp67yEhTi17ZDBBC1z9pTdGuzjD+EFIqLSYRweZjqfiPzQ06Ebg==} engines: {node: '>= 0.4'} + get-tsconfig@5.0.0-beta.5: + resolution: {integrity: sha512-/6gFNr0N04nob252sTQxyFLi3eKFRqIg1I87YcqAMT1i6SQrSF6KujUEQrtrjMV0H/eejTCltLdDSTEMzHbnsQ==} + engines: {node: '>=20.20.0'} + glob-parent@6.0.2: resolution: {integrity: sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==} engines: {node: '>=10.13.0'} @@ -1846,6 +1912,11 @@ packages: resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} hasBin: true + jsesc@3.1.0: + resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} + engines: {node: '>=6'} + hasBin: true + json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -2213,6 +2284,9 @@ packages: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + resolve@2.0.0-next.6: resolution: {integrity: sha512-3JmVl5hMGtJ3kMmB3zi3DL25KfkCEyy3Tw7Gmw7z5w8M9WlwoPFnIvwChzu1+cF3iaK3sp18hhPz8ANeimdJfA==} engines: {node: '>= 0.4'} @@ -2223,6 +2297,25 @@ packages: engines: {node: 20 || >=22} hasBin: true + rolldown-plugin-dts@0.25.2: + resolution: {integrity: sha512-nMhN/R+vmR8GM45ZW1FWMSjRTSDDn/6w4GTf8RNrEFCBdl8B1kySWrU1ixPtbwzXoRlcO+R/S88VgXuJQwfdDg==} + engines: {node: ^22.18.0 || >=24.0.0} + peerDependencies: + '@ts-macro/tsc': ^0.3.6 + '@typescript/native-preview': '>=7.0.0-dev.20260325.1' + rolldown: ^1.0.0 + typescript: ^5.0.0 || ^6.0.0 + vue-tsc: ~3.2.0 + peerDependenciesMeta: + '@ts-macro/tsc': + optional: true + '@typescript/native-preview': + optional: true + typescript: + optional: true + vue-tsc: + optional: true + rolldown@1.0.1: resolution: {integrity: sha512-X0KQHljNnEkWNqqiz9zJrGunh1B0HgOxLXvnFpCOcadzcy5qohZ3tqMEUg00vncoRovXuK3ZqCT9KnnKzoInFQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2647,8 +2740,36 @@ packages: snapshots: + '@babel/generator@8.0.0-rc.6': + dependencies: + '@babel/parser': 8.0.0-rc.6 + '@babel/types': 8.0.0 + '@jridgewell/gen-mapping': 0.3.13 + '@jridgewell/trace-mapping': 0.3.31 + '@types/jsesc': 2.5.1 + jsesc: 3.1.0 + + '@babel/helper-string-parser@8.0.0': {} + + '@babel/helper-validator-identifier@8.0.0-rc.6': {} + + '@babel/helper-validator-identifier@8.0.2': {} + + '@babel/parser@8.0.0': + dependencies: + '@babel/types': 8.0.0 + + '@babel/parser@8.0.0-rc.6': + dependencies: + '@babel/types': 8.0.0 + '@babel/runtime@7.29.7': {} + '@babel/types@8.0.0': + dependencies: + '@babel/helper-string-parser': 8.0.0 + '@babel/helper-validator-identifier': 8.0.2 + '@balena/dockerignore@1.0.2': {} '@emnapi/core@1.10.0': @@ -2861,8 +2982,20 @@ snapshots: '@img/sharp-win32-x64@0.34.5': optional: true + '@jridgewell/gen-mapping@0.3.13': + dependencies: + '@jridgewell/sourcemap-codec': 1.5.5 + '@jridgewell/trace-mapping': 0.3.31 + + '@jridgewell/resolve-uri@3.1.2': {} + '@jridgewell/sourcemap-codec@1.5.5': {} + '@jridgewell/trace-mapping@0.3.31': + dependencies: + '@jridgewell/resolve-uri': 3.1.2 + '@jridgewell/sourcemap-codec': 1.5.5 + '@js-sdsl/ordered-map@4.4.2': {} '@lit-labs/react@1.2.1': {} @@ -3082,6 +3215,8 @@ snapshots: '@types/estree@1.0.9': {} + '@types/jsesc@2.5.1': {} + '@types/json-schema@7.0.15': {} '@types/json5@0.0.29': {} @@ -3496,6 +3631,12 @@ snapshots: assertion-error@2.0.1: {} + ast-kit@3.0.0: + dependencies: + '@babel/parser': 8.0.0 + estree-walker: 3.0.3 + pathe: 2.0.3 + ast-types-flow@0.0.8: {} async-function@1.0.0: {} @@ -3554,6 +3695,8 @@ snapshots: dependencies: tweetnacl: 0.14.5 + birpc@4.0.0: {} + bl@4.1.0: dependencies: buffer: 5.7.1 @@ -3721,6 +3864,8 @@ snapshots: dependencies: esutils: 2.0.3 + dts-resolver@3.0.0: {} + dunder-proto@1.0.1: dependencies: call-bind-apply-helpers: 1.0.2 @@ -4127,6 +4272,10 @@ snapshots: es-errors: 1.3.0 get-intrinsic: 1.3.0 + get-tsconfig@5.0.0-beta.5: + dependencies: + resolve-pkg-maps: 1.0.0 + glob-parent@6.0.2: dependencies: is-glob: 4.0.3 @@ -4315,6 +4464,8 @@ snapshots: dependencies: argparse: 2.0.1 + jsesc@3.1.0: {} + json-buffer@3.0.1: {} json-schema-traverse@0.4.1: {} @@ -4665,6 +4816,8 @@ snapshots: resolve-from@4.0.0: {} + resolve-pkg-maps@1.0.0: {} + resolve@2.0.0-next.6: dependencies: es-errors: 1.3.0 @@ -4679,6 +4832,22 @@ snapshots: glob: 13.0.6 package-json-from-dist: 1.0.1 + rolldown-plugin-dts@0.25.2(rolldown@1.0.1)(typescript@6.0.3): + dependencies: + '@babel/generator': 8.0.0-rc.6 + '@babel/helper-validator-identifier': 8.0.0-rc.6 + '@babel/parser': 8.0.0-rc.6 + ast-kit: 3.0.0 + birpc: 4.0.0 + dts-resolver: 3.0.0 + get-tsconfig: 5.0.0-beta.5 + obug: 2.1.1 + rolldown: 1.0.1 + optionalDependencies: + typescript: 6.0.3 + transitivePeerDependencies: + - oxc-resolver + rolldown@1.0.1: dependencies: '@oxc-project/types': 0.130.0 From b528b96fc6387d77f0bc44b3fb229ce01c335612 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 30 Jun 2026 16:12:31 +0000 Subject: [PATCH 4/4] feat: support experiment file paths --- README.md | 6 ++++++ packages/agent-eval/README.md | 4 +++- packages/agent-eval/src/experiments.test.ts | 16 ++++++++++++++ packages/agent-eval/src/experiments.ts | 23 ++++++++++++++------- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e384f4e..e3c96ae 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,12 @@ Run local experiments with the `agent-eval` CLI: COPILOT_GITHUB_TOKEN=... agent-eval --experiments ./experiments --experiment example ``` +You can also provide a path directly to `--experiment`: + +```sh +COPILOT_GITHUB_TOKEN=... agent-eval --experiment ./experiments/example.ts +``` + The experiment config will specify: - A name for the experiment diff --git a/packages/agent-eval/README.md b/packages/agent-eval/README.md index 340b1a9..e5c130f 100644 --- a/packages/agent-eval/README.md +++ b/packages/agent-eval/README.md @@ -11,7 +11,9 @@ COPILOT_GITHUB_TOKEN=... agent-eval --experiments ./experiments --experiment exa ``` Use `--experiments` to load experiment files from a local directory. Experiment -files may export an `experiment` named export or a default export. +files may export an `experiment` named export or a default export. `--experiment` +may also be a path to a local experiment file when you only want to run one +experiment. ## Experiment config authoring diff --git a/packages/agent-eval/src/experiments.test.ts b/packages/agent-eval/src/experiments.test.ts index be5321d..1e152ef 100644 --- a/packages/agent-eval/src/experiments.test.ts +++ b/packages/agent-eval/src/experiments.test.ts @@ -48,6 +48,22 @@ describe('local experiment loading', () => { ) }) + test('finds an experiment from a local file path', async () => { + const directory = await createExperimentsDirectory() + + await expect(findExperiment(path.join(directory, 'example.mjs'))).resolves.toEqual( + expect.objectContaining({name: 'Example'}), + ) + }) + + test('loads an experiment from a local file path', async () => { + const directory = await createExperimentsDirectory() + + await expect(loadExperimentConfigs({experiment: path.join(directory, 'example.mjs')})).resolves.toEqual([ + expect.objectContaining({name: 'Example'}), + ]) + }) + test('loads all experiments when no experiment is specified', async () => { const directory = await createExperimentsDirectory() diff --git a/packages/agent-eval/src/experiments.ts b/packages/agent-eval/src/experiments.ts index 04a957b..bf9bf61 100644 --- a/packages/agent-eval/src/experiments.ts +++ b/packages/agent-eval/src/experiments.ts @@ -32,6 +32,17 @@ function isExperimentFile(filename: string): boolean { return filename !== 'index.ts' && EXPERIMENT_FILE_EXTENSIONS.has(path.extname(filename)) } +async function loadExperimentFile(filepath: string): Promise { + const resolvedPath = path.resolve(filepath) + const mod = (await import(pathToFileURL(resolvedPath).href)) as ExperimentModule + const experiment = mod.experiment ?? mod.default + if (!experiment) { + throw new Error(`Experiment file must export "experiment" or a default export: ${resolvedPath}`) + } + + return experiment +} + async function getLocalExperimentEntries(experimentsDirectory: string): Promise> { const directory = path.resolve(experimentsDirectory) if (!existsSync(directory)) { @@ -47,13 +58,7 @@ async function getLocalExperimentEntries(experimentsDirectory: string): Promise< return Promise.all( filenames.filter(isExperimentFile).map(async filename => { const filepath = path.join(directory, filename) - const mod = (await import(pathToFileURL(filepath).href)) as ExperimentModule - const experiment = mod.experiment ?? mod.default - if (!experiment) { - throw new Error(`Experiment file must export "experiment" or a default export: ${filepath}`) - } - - return [getExperimentId(filename), experiment] + return [getExperimentId(filename), await loadExperimentFile(filepath)] }), ) } @@ -75,6 +80,10 @@ async function findExperiment( return experiments.find(([name]) => name === id)?.[1] } + if (existsSync(id)) { + return loadExperimentFile(id) + } + return findPackagedExperiment(id) }