Skip to content

Commit 227d0fd

Browse files
rlagentflowbeta[bot]Runloop Agentclaude
authored
perf(cli): parallelize scenario run fetching and name resolution in bmj logs (#179)
## Summary - Replace the serial per-agent loop in \`downloadBenchmarkJobLogs\` with \`Promise.allSettled\`, so all agents' \`listBenchmarkRunScenarioRuns\` calls run concurrently - Replace serial \`resolveScenarioName\` calls with a single \`Promise.all\` batch across all scenario runs The log downloads were already parallelized (max concurrency 50). This PR fixes the loading/setup phase which was the remaining serial bottleneck. **Before**: loading time = sum of all agents' fetch times + sum of all scenario name lookups **After**: loading time ~= slowest single agent fetch + slowest single name lookup ## Test plan - [ ] Run \`rli bmj logs <job-id>\` with a multi-agent benchmark job and verify logs download correctly - [ ] Verify agent fetch failures surface as warnings (not crashes) via the \`allSettled\` error handling - [ ] Verify \`--run\` and \`--scenario\` filters still work correctly 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Runloop Agent <agent@runloop.ai> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent d8b35a2 commit 227d0fd

1 file changed

Lines changed: 46 additions & 28 deletions

File tree

src/commands/benchmark-job/logs.ts

Lines changed: 46 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -302,41 +302,59 @@ export async function downloadBenchmarkJobLogs(
302302
// Build scenario outcome lookup from completed outcomes
303303
const outcomeMap = buildScenarioOutcomeMap(job);
304304

305-
// Gather all scenario log targets across benchmark runs
306-
const targets: ScenarioLogTarget[] = [];
307-
308-
for (const run of runs) {
309-
const agentLabel = run.modelName
310-
? `${run.agentName}:${run.modelName}`
311-
: run.agentName;
312-
console.log(
313-
chalk.dim(`Fetching scenario runs for agent "${agentLabel}"...`),
314-
);
315-
let scenarioRuns = await listBenchmarkRunScenarioRuns(run.benchmarkRunId);
316-
317-
// Apply --scenario filter
305+
// Gather all scenario log targets across benchmark runs, fetching all
306+
// agents' scenario run lists in parallel for speed.
307+
console.log(
308+
chalk.dim(
309+
`Fetching scenario runs for ${runs.length} agent(s) in parallel...`,
310+
),
311+
);
312+
313+
const agentScenarioRuns = await Promise.allSettled(
314+
runs.map((run) => listBenchmarkRunScenarioRuns(run.benchmarkRunId)),
315+
);
316+
317+
// Collect (run, scenarioRun) pairs, applying --scenario filter
318+
const pairs: { run: BenchmarkRunInfo; sr: ScenarioRun }[] = [];
319+
for (let i = 0; i < runs.length; i++) {
320+
const result = agentScenarioRuns[i];
321+
if (result.status === "rejected") {
322+
const agentLabel = runs[i].modelName
323+
? `${runs[i].agentName}:${runs[i].modelName}`
324+
: runs[i].agentName;
325+
console.error(
326+
chalk.yellow(
327+
` Warning: failed to fetch scenario runs for agent "${agentLabel}": ${result.reason}`,
328+
),
329+
);
330+
continue;
331+
}
332+
let scenarioRuns = result.value;
318333
if (options.scenario) {
319334
scenarioRuns = scenarioRuns.filter((sr) => sr.id === options.scenario);
320335
}
321-
322336
for (const sr of scenarioRuns) {
323-
const scenarioName = await resolveScenarioName(
324-
sr.id,
325-
sr.scenario_id,
326-
outcomeMap,
327-
);
328-
targets.push({
329-
agentName: run.agentName,
330-
modelName: run.modelName,
331-
scenarioName,
332-
scenarioRunId: sr.id,
333-
scenarioRun: sr,
334-
outcome: outcomeMap.get(sr.id),
335-
destDir: "", // assigned below
336-
});
337+
pairs.push({ run: runs[i], sr });
337338
}
338339
}
339340

341+
// Resolve all scenario names in parallel (may hit the API for in-progress runs)
342+
const resolvedNames = await Promise.all(
343+
pairs.map(({ sr }) =>
344+
resolveScenarioName(sr.id, sr.scenario_id, outcomeMap),
345+
),
346+
);
347+
348+
const targets: ScenarioLogTarget[] = pairs.map(({ run, sr }, i) => ({
349+
agentName: run.agentName,
350+
modelName: run.modelName,
351+
scenarioName: resolvedNames[i],
352+
scenarioRunId: sr.id,
353+
scenarioRun: sr,
354+
outcome: outcomeMap.get(sr.id),
355+
destDir: "", // assigned below
356+
}));
357+
340358
if (targets.length === 0) {
341359
console.log(chalk.yellow("No scenario runs found to download logs for."));
342360
return;

0 commit comments

Comments
 (0)