Skip to content

Commit 2eb991b

Browse files
iamzifeiclaude
iamzifei
and
claude
committed
fix: extract eval engine + in-process batch runner
Root cause found: execFileSync child process signal handling was killing MCP server processes prematurely. The MCP connection itself is fine (tested: survives 60s+ idle with no issues). Changes: - Extract core eval pipeline into eval/engine.ts (reusable by CLI and batch) - CLI run command now delegates to engine.runEvaluation() - Batch runner imports engine directly — no child process spawning - Add src/index.ts as library entry point for programmatic use - tsup now builds both cli.js and index.js Verified: mcp-memory scores 78/100 via in-process batch runner. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 5b21b25 commit 2eb991b

5 files changed

Lines changed: 276 additions & 211 deletions

File tree

‎packages/agent-eval/src/commands/run.ts‎

Lines changed: 35 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@ import chalk from "chalk";
22
import { Command } from "commander";
33
import ora from "ora";
44
import { loadConfig } from "../config/loader.js";
5-
import { executeTasks } from "../eval/executor.js";
6-
import { scoreResults } from "../eval/scorer.js";
7-
import { generateTasks } from "../eval/task-generator.js";
8-
import { createAdapter } from "../protocols/factory.js";
9-
import { buildReport, printReport, saveReport } from "../report/generator.js";
5+
import { runEvaluation } from "../eval/engine.js";
6+
import { printReport } from "../report/generator.js";
107

118
/**
129
* Load API key from environment.
@@ -50,106 +47,42 @@ export const runCommand = new Command("run")
5047
)
5148
.action(async (options) => {
5249
try {
53-
await runEval(options);
50+
// Load config
51+
const config = loadConfig(options.config);
52+
const apiKey = await getApiKey();
53+
54+
const runsPerTask = options.runs
55+
? Number.parseInt(options.runs, 10)
56+
: config.eval.runs;
57+
const tasksPerTool = Number.parseInt(options.tasksPerTool, 10);
58+
59+
// Run evaluation with spinner progress
60+
const spinner = ora().start();
61+
const result = await runEvaluation({
62+
config,
63+
apiKey,
64+
tasksPerTool,
65+
runsPerTask,
66+
outputDir: options.output,
67+
onProgress: (_step, detail) => {
68+
spinner.text = detail;
69+
},
70+
});
71+
spinner.succeed("Evaluation complete");
72+
73+
// Display results
74+
if (options.json) {
75+
console.log(JSON.stringify(result.report, null, 2));
76+
} else {
77+
printReport(result.report);
78+
if (result.reportPath) {
79+
console.log(chalk.dim(` Full report: ${result.reportPath}`));
80+
console.log("");
81+
}
82+
}
5483
} catch (err) {
5584
const message = err instanceof Error ? err.message : String(err);
5685
console.error(chalk.red(`\nError: ${message}`));
5786
process.exit(1);
5887
}
5988
});
60-
61-
async function runEval(options: {
62-
config: string;
63-
runs?: string;
64-
tasksPerTool: string;
65-
json?: boolean;
66-
output: string;
67-
}) {
68-
// Step 1: Load config
69-
const spinner = ora("Loading config...").start();
70-
const config = loadConfig(options.config);
71-
const runsPerTask = options.runs
72-
? Number.parseInt(options.runs, 10)
73-
: config.eval.runs;
74-
const tasksPerTool = Number.parseInt(options.tasksPerTool, 10);
75-
spinner.succeed(
76-
`Config loaded: ${config.agent.name} (${config.agent.protocol.toUpperCase()})`,
77-
);
78-
79-
// Step 2: Get API key
80-
const apiKey = await getApiKey();
81-
82-
// Step 3: Connect and discover tools.
83-
// We keep one connection open for the entire eval — discovery, task gen, and execution.
84-
spinner.start(
85-
`Connecting to agent via ${config.agent.protocol.toUpperCase()}...`,
86-
);
87-
const adapter = createAdapter(config.agent.protocol, config.agent.endpoint);
88-
await adapter.connect();
89-
const tools = await adapter.listTools();
90-
spinner.succeed(`Discovered ${tools.length} tool(s)`);
91-
92-
if (tools.length === 0) {
93-
console.log(chalk.yellow("No tools found. Nothing to evaluate."));
94-
await adapter.disconnect().catch(() => {});
95-
return;
96-
}
97-
98-
// Step 4: Generate test tasks using Claude API
99-
spinner.start(
100-
`Generating ${tasksPerTool} test tasks per tool (${tools.length} tools)...`,
101-
);
102-
const tasks = await generateTasks(tools, config.agent.capabilities, {
103-
tasksPerTool,
104-
apiKey,
105-
});
106-
spinner.succeed(`Generated ${tasks.length} test tasks`);
107-
108-
// Step 5: Execute tasks (reuse the same connection)
109-
const totalRuns = tasks.length * runsPerTask;
110-
spinner.start(
111-
`Executing ${totalRuns} runs (${tasks.length} tasks x ${runsPerTask} runs)...`,
112-
);
113-
const execution = await executeTasks(adapter, tasks, runsPerTask, {
114-
onRunComplete: (completed, total) => {
115-
spinner.text = `Executing runs... ${completed}/${total}`;
116-
},
117-
});
118-
spinner.succeed(
119-
`Executed ${execution.totalRuns} runs (${execution.totalSuccessful} successful)`,
120-
);
121-
122-
// Step 6: Disconnect
123-
await adapter.disconnect().catch(() => {});
124-
125-
// Step 8: Score results with LLM-as-judge
126-
spinner.start("Scoring results with LLM-as-judge...");
127-
const weights = {
128-
capability: config.eval.dimensions.capability.weight,
129-
reliability: config.eval.dimensions.reliability.weight,
130-
efficiency: config.eval.dimensions.efficiency.weight,
131-
safety: config.eval.dimensions.safety.weight,
132-
developer_experience: config.eval.dimensions.developer_experience.weight,
133-
};
134-
const scores = await scoreResults(execution.taskResults, {
135-
apiKey,
136-
weights,
137-
onTaskScored: (completed, total) => {
138-
spinner.text = `Scoring tasks... ${completed}/${total}`;
139-
},
140-
});
141-
spinner.succeed("Scoring complete");
142-
143-
// Step 9: Generate and save report
144-
const report = buildReport(config, tools, execution, scores);
145-
const reportPath = saveReport(report, options.output);
146-
147-
// Step 10: Display results
148-
if (options.json) {
149-
console.log(JSON.stringify(report, null, 2));
150-
} else {
151-
printReport(report);
152-
console.log(chalk.dim(` Full report: ${reportPath}`));
153-
console.log("");
154-
}
155-
}

‎packages/agent-eval/src/eval/engine.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/**
2+
* Core evaluation engine — orchestrates the full eval pipeline.
3+
* Used by both the CLI `run` command and the batch runner script.
4+
*/
5+
6+
import type { AgentEvalConfig } from "../config/schema.js";
7+
import type { ToolInfo } from "../protocols/base.js";
8+
import { createAdapter } from "../protocols/factory.js";
9+
import {
10+
buildReport,
11+
type EvalReport,
12+
saveReport,
13+
} from "../report/generator.js";
14+
import { executeTasks } from "./executor.js";
15+
import { scoreResults } from "./scorer.js";
16+
import { generateTasks } from "./task-generator.js";
17+
18+
/** Progress callback — receives step name and detail message */
19+
export type ProgressCallback = (step: string, detail: string) => void;
20+
21+
/** Options for running an evaluation */
22+
export interface EvalOptions {
23+
config: AgentEvalConfig;
24+
apiKey: string;
25+
tasksPerTool?: number;
26+
runsPerTask?: number;
27+
outputDir?: string;
28+
/** Called on each major step for progress reporting */
29+
onProgress?: ProgressCallback;
30+
}
31+
32+
/** Result of a full evaluation run */
33+
export interface EvalResult {
34+
report: EvalReport;
35+
reportPath?: string;
36+
}
37+
38+
/**
39+
* Run a full evaluation pipeline:
40+
* 1. Connect to agent via protocol adapter
41+
* 2. Discover tools
42+
* 3. Generate test tasks (Claude API)
43+
* 4. Execute tasks and collect metrics
44+
* 5. Score results with LLM-as-judge
45+
* 6. Build and save report
46+
*
47+
* This function is self-contained and manages its own MCP connections.
48+
*/
49+
export async function runEvaluation(options: EvalOptions): Promise<EvalResult> {
50+
const {
51+
config,
52+
apiKey,
53+
tasksPerTool = 3,
54+
runsPerTask = config.eval.runs,
55+
outputDir,
56+
onProgress,
57+
} = options;
58+
59+
const progress = onProgress ?? (() => {});
60+
61+
// Step 1: Connect and discover tools
62+
progress(
63+
"connect",
64+
`Connecting via ${config.agent.protocol.toUpperCase()}...`,
65+
);
66+
const adapter = createAdapter(config.agent.protocol, config.agent.endpoint);
67+
await adapter.connect();
68+
69+
let tools: ToolInfo[];
70+
try {
71+
tools = await adapter.listTools();
72+
} catch (err) {
73+
await adapter.disconnect().catch(() => {});
74+
throw err;
75+
}
76+
77+
progress("discover", `Discovered ${tools.length} tool(s)`);
78+
79+
if (tools.length === 0) {
80+
await adapter.disconnect().catch(() => {});
81+
throw new Error("No tools found. Nothing to evaluate.");
82+
}
83+
84+
// Step 2: Generate test tasks using Claude API
85+
// The MCP connection stays open during this step — tested to survive 60s+ idle
86+
progress(
87+
"generate",
88+
`Generating ${tasksPerTool} tasks per tool (${tools.length} tools)...`,
89+
);
90+
const tasks = await generateTasks(tools, config.agent.capabilities, {
91+
tasksPerTool,
92+
apiKey,
93+
});
94+
progress("generate", `Generated ${tasks.length} test tasks`);
95+
96+
// Step 3: Execute tasks
97+
const totalRuns = tasks.length * runsPerTask;
98+
progress("execute", `Executing ${totalRuns} runs...`);
99+
const execution = await executeTasks(adapter, tasks, runsPerTask, {
100+
onRunComplete: (completed, total) => {
101+
progress("execute", `Executing runs... ${completed}/${total}`);
102+
},
103+
});
104+
progress(
105+
"execute",
106+
`Executed ${execution.totalRuns} runs (${execution.totalSuccessful} successful)`,
107+
);
108+
109+
// Step 4: Disconnect
110+
await adapter.disconnect().catch(() => {});
111+
112+
// Step 5: Score results
113+
progress("score", "Scoring with LLM-as-judge...");
114+
const weights = {
115+
capability: config.eval.dimensions.capability.weight,
116+
reliability: config.eval.dimensions.reliability.weight,
117+
efficiency: config.eval.dimensions.efficiency.weight,
118+
safety: config.eval.dimensions.safety.weight,
119+
developer_experience: config.eval.dimensions.developer_experience.weight,
120+
};
121+
const scores = await scoreResults(execution.taskResults, {
122+
apiKey,
123+
weights,
124+
onTaskScored: (completed, total) => {
125+
progress("score", `Scoring tasks... ${completed}/${total}`);
126+
},
127+
});
128+
progress("score", "Scoring complete");
129+
130+
// Step 6: Build report
131+
const report = buildReport(config, tools, execution, scores);
132+
133+
let reportPath: string | undefined;
134+
if (outputDir) {
135+
reportPath = saveReport(report, outputDir);
136+
}
137+
138+
return { report, reportPath };
139+
}

‎packages/agent-eval/src/index.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
// Public API for programmatic use (batch runner, etc.)
2+
3+
export { loadConfig } from "./config/loader.js";
4+
export type { AgentEvalConfig } from "./config/schema.js";
5+
export type {
6+
EvalOptions,
7+
EvalResult,
8+
ProgressCallback,
9+
} from "./eval/engine.js";
10+
export { runEvaluation } from "./eval/engine.js";
11+
export type { EvalReport } from "./report/generator.js";

‎packages/agent-eval/tsup.config.ts‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,24 @@
11
import { defineConfig } from "tsup";
22

3-
export default defineConfig({
4-
entry: ["src/cli.ts"],
5-
format: ["esm"],
6-
target: "node20",
7-
outDir: "dist",
8-
clean: true,
9-
sourcemap: true,
10-
// Add shebang for CLI binary
11-
banner: {
12-
js: "#!/usr/bin/env node",
3+
export default defineConfig([
4+
// CLI binary
5+
{
6+
entry: ["src/cli.ts"],
7+
format: ["esm"],
8+
target: "node20",
9+
outDir: "dist",
10+
clean: true,
11+
sourcemap: true,
12+
banner: {
13+
js: "#!/usr/bin/env node",
14+
},
1315
},
14-
});
16+
// Library entry (for programmatic use by batch runner, etc.)
17+
{
18+
entry: ["src/index.ts"],
19+
format: ["esm"],
20+
target: "node20",
21+
outDir: "dist",
22+
sourcemap: true,
23+
},
24+
]);

0 commit comments

Comments
 (0)