Skip to content

Commit 32f4f72

Browse files
authored
feat(benchmark): print scenario runtime, status and prompt for viewing extended (#152)
## Description Command Rename: status → summary - Renamed benchmark-job status to benchmark-job summary - File renamed: status.ts → summary.ts New -e, --extended Flag for Summary - By default, rli benchmark-job summary <id> shows only the aggregate table (agent scores) - With -e flag, shows individual scenario results underneath each agent Watch Command Improvements Elapsed time per scenario: - Each running scenario now shows how long it's been running (e.g., running (2m 30s)) Completion behavior: - On job completion, exits full-screen and shows: - "Benchmark job completed!" message - Total elapsed time - Summary table (without individual scenarios) - Hint: To see full results, run: rli benchmark-job summary -e <job_id> Display improvements: - Added job ID line under the job name header - Added terminal resize handler to prevent display artifacts - Content now truncates to fit terminal height (header stays visible, progress truncated from bottom) ## Type of Change <!-- Mark the relevant option with an 'x' --> - [ ] Bug fix (non-breaking change which fixes an issue) - [x] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation update - [ ] Code refactoring - [ ] Performance improvement - [ ] Test updates ## Related Issues <!-- Link to related issues using #issue-number --> Closes # ## Changes Made <!-- Describe the changes in detail --> ## Testing <!-- Describe how you tested your changes --> - [ ] I have tested locally - [ ] I have added/updated tests - [ ] All existing tests pass ## Checklist - [ ] My code follows the code style of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published ## Screenshots (if applicable) <!-- Add screenshots to help explain your changes --> ## Additional Notes <!-- Any additional information that reviewers should know -->
1 parent c49ee7a commit 32f4f72

4 files changed

Lines changed: 128 additions & 97 deletions

File tree

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ rli mcp install # Install Runloop MCP server configurat
185185

186186
```bash
187187
rli benchmark-job run # Run a benchmark job with one or more ...
188-
rli benchmark-job status <id> # Get benchmark job status and results
188+
rli benchmark-job summary <id> # Get benchmark job summary and results
189189
rli benchmark-job watch <id> # Watch benchmark job progress in real-...
190190
```
191191

Lines changed: 49 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Status benchmark job command
2+
* Summary benchmark job command
33
*/
44

55
import chalk from "chalk";
@@ -11,8 +11,9 @@ import {
1111
} from "../../services/benchmarkJobService.js";
1212
import { output, outputError } from "../../utils/output.js";
1313

14-
interface StatusOptions {
14+
interface SummaryOptions {
1515
output?: string;
16+
extended?: boolean;
1617
}
1718

1819
// Job states that indicate completion
@@ -285,7 +286,7 @@ function calculateCompletedStats(
285286
}
286287

287288
// Print results table for completed jobs
288-
function printResultsTable(job: BenchmarkJob): void {
289+
function printResultsTable(job: BenchmarkJob, extended: boolean = false): void {
289290
const outcomes = job.benchmark_outcomes || [];
290291

291292
if (outcomes.length === 0) {
@@ -363,55 +364,59 @@ function printResultsTable(job: BenchmarkJob): void {
363364
chalk.dim(totalColStr),
364365
);
365366

366-
// Print individual scenario results underneath (indented)
367-
for (const scenario of scenarioOutcomes) {
368-
const scenarioName =
369-
scenario.scenario_name || scenario.scenario_definition_id || "unknown";
370-
const state = scenario.state || "unknown";
371-
const score = scenario.score;
372-
373-
let statusIcon: string;
374-
let statusColor: typeof chalk.green;
375-
376-
if (state.toUpperCase() === "COMPLETED") {
377-
if (score === 1.0) {
378-
statusIcon = chalk.green("\u2713"); // checkmark
379-
statusColor = chalk.green;
367+
// Print individual scenario results underneath (indented) when extended
368+
if (extended) {
369+
for (const scenario of scenarioOutcomes) {
370+
const scenarioName =
371+
scenario.scenario_name ||
372+
scenario.scenario_definition_id ||
373+
"unknown";
374+
const state = scenario.state || "unknown";
375+
const score = scenario.score;
376+
377+
let statusIcon: string;
378+
let statusColor: typeof chalk.green;
379+
380+
if (state.toUpperCase() === "COMPLETED") {
381+
if (score === 1.0) {
382+
statusIcon = chalk.green("\u2713"); // checkmark
383+
statusColor = chalk.green;
384+
} else {
385+
statusIcon = chalk.yellow("\u2717"); // X
386+
statusColor = chalk.yellow;
387+
}
380388
} else {
381-
statusIcon = chalk.yellow("\u2717"); // X
382-
statusColor = chalk.yellow;
389+
statusIcon = chalk.red("!");
390+
statusColor = chalk.red;
383391
}
384-
} else {
385-
statusIcon = chalk.red("!");
386-
statusColor = chalk.red;
387-
}
388392

389-
const scenarioNameTrunc =
390-
scenarioName.length > 50
391-
? scenarioName.slice(0, 47) + "..."
392-
: scenarioName;
393-
394-
const scoreStr =
395-
score !== undefined && score !== null
396-
? `score=${score.toFixed(1)}`
397-
: state;
398-
399-
console.log(
400-
chalk.dim(" ") +
401-
statusIcon +
402-
" " +
403-
chalk.dim(scenarioNameTrunc.padEnd(52)) +
404-
statusColor(scoreStr),
405-
);
393+
const scenarioNameTrunc =
394+
scenarioName.length > 50
395+
? scenarioName.slice(0, 47) + "..."
396+
: scenarioName;
397+
398+
const scoreStr =
399+
score !== undefined && score !== null
400+
? `score=${score.toFixed(1)}`
401+
: state;
402+
403+
console.log(
404+
chalk.dim(" ") +
405+
statusIcon +
406+
" " +
407+
chalk.dim(scenarioNameTrunc.padEnd(52)) +
408+
statusColor(scoreStr),
409+
);
410+
}
406411
}
407412
}
408413

409414
console.log();
410415
}
411416

412-
export async function statusBenchmarkJob(
417+
export async function summaryBenchmarkJob(
413418
id: string,
414-
options: StatusOptions = {},
419+
options: SummaryOptions = {},
415420
) {
416421
try {
417422
const job = await getBenchmarkJob(id);
@@ -420,11 +425,11 @@ export async function statusBenchmarkJob(
420425
if (options.output && options.output !== "text") {
421426
output(job, { format: options.output, defaultFormat: "json" });
422427
} else if (isComplete) {
423-
printResultsTable(job);
428+
printResultsTable(job, options.extended);
424429
} else {
425430
await printStatus(job);
426431
}
427432
} catch (error) {
428-
outputError("Failed to get benchmark job status", error);
433+
outputError("Failed to get benchmark job summary", error);
429434
}
430435
}

src/commands/benchmark-job/watch.ts

Lines changed: 72 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,13 @@ function exitFullScreen(): void {
5656
process.stdout.write(ANSI.exitAltScreen);
5757
}
5858

59-
// Render content at top of screen
59+
// Render content at top of screen, truncating to fit terminal height
6060
function renderScreen(lines: string[]): void {
6161
process.stdout.write(ANSI.moveTo(1, 1));
6262
process.stdout.write(ANSI.clearScreen);
63-
for (const line of lines) {
63+
const maxLines = (process.stdout.rows || 24) - 1; // Leave 1 line buffer
64+
const truncatedLines = lines.slice(0, maxLines);
65+
for (const line of truncatedLines) {
6466
console.log(line);
6567
}
6668
}
@@ -71,10 +73,27 @@ function formatPercent(count: number, total: number): string {
7173
return ((count / total) * 100).toFixed(1) + "%";
7274
}
7375

76+
// Format duration in human-readable format
77+
function formatDuration(ms: number): string {
78+
const seconds = Math.floor(ms / 1000);
79+
const minutes = Math.floor(seconds / 60);
80+
const hours = Math.floor(minutes / 60);
81+
const remainingMinutes = minutes % 60;
82+
const remainingSeconds = seconds % 60;
83+
if (hours > 0) {
84+
return `${hours}h ${remainingMinutes}m ${remainingSeconds}s`;
85+
}
86+
if (minutes > 0) {
87+
return `${minutes}m ${remainingSeconds}s`;
88+
}
89+
return `${seconds}s`;
90+
}
91+
7492
// In-progress scenario info for display
7593
interface InProgressScenario {
7694
name: string;
7795
state: string;
96+
startTimeMs?: number;
7897
}
7998

8099
// Progress stats for a benchmark run
@@ -141,17 +160,20 @@ function calculateRunProgress(
141160
inProgressScenarios.push({
142161
name: scenario.name || scenario.scenario_id || "unknown",
143162
state: scenarioState,
163+
startTimeMs: scenario.start_time_ms,
144164
});
145165
} else if (scenarioState === "running") {
146166
running++;
147167
inProgressScenarios.push({
148168
name: scenario.name || scenario.scenario_id || "unknown",
149169
state: scenarioState,
170+
startTimeMs: scenario.start_time_ms,
150171
});
151172
} else if (scenarioState && scenarioState !== "pending") {
152173
inProgressScenarios.push({
153174
name: scenario.name || scenario.scenario_id || "unknown",
154175
state: scenarioState,
176+
startTimeMs: scenario.start_time_ms,
155177
});
156178
}
157179
}
@@ -311,15 +333,29 @@ function formatScenarioLines(
311333
): string[] {
312334
const limited = scenarios.slice(0, maxScenarios);
313335
const remaining = scenarios.length - limited.length;
336+
const now = Date.now();
314337

315338
const lines: string[] = [];
316339
for (const scenario of limited) {
317340
let name = scenario.name;
318-
if (name.length > 45) {
319-
name = name.slice(0, 42) + "...";
341+
if (name.length > 35) {
342+
name = name.slice(0, 32) + "...";
320343
}
321344
const stateStr = formatScenarioState(scenario.state, tick);
322-
lines.push(chalk.dim(" └ ") + chalk.dim(name.padEnd(46)) + stateStr);
345+
346+
// Calculate elapsed time if start time is available
347+
let elapsedStr = "";
348+
if (scenario.startTimeMs) {
349+
const elapsedMs = now - scenario.startTimeMs;
350+
elapsedStr = chalk.dim(` (${formatDuration(elapsedMs)})`);
351+
}
352+
353+
lines.push(
354+
chalk.dim(" └ ") +
355+
chalk.dim(name.padEnd(36)) +
356+
stateStr +
357+
elapsedStr,
358+
);
323359
}
324360

325361
if (remaining > 0) {
@@ -457,47 +493,6 @@ function printResultsTable(job: BenchmarkJob): void {
457493
failedErrorColored +
458494
chalk.dim(totalColStr),
459495
);
460-
461-
for (const scenario of scenarioOutcomes) {
462-
const scenarioName =
463-
scenario.scenario_name || scenario.scenario_definition_id || "unknown";
464-
const state = scenario.state || "unknown";
465-
const score = scenario.score;
466-
467-
let statusIcon: string;
468-
let statusColor: typeof chalk.green;
469-
470-
if (state.toUpperCase() === "COMPLETED") {
471-
if (score === 1.0) {
472-
statusIcon = chalk.green("\u2713");
473-
statusColor = chalk.green;
474-
} else {
475-
statusIcon = chalk.yellow("\u2717");
476-
statusColor = chalk.yellow;
477-
}
478-
} else {
479-
statusIcon = chalk.red("!");
480-
statusColor = chalk.red;
481-
}
482-
483-
const scenarioNameTrunc =
484-
scenarioName.length > 50
485-
? scenarioName.slice(0, 47) + "..."
486-
: scenarioName;
487-
488-
const scoreStr =
489-
score !== undefined && score !== null
490-
? `score=${score.toFixed(1)}`
491-
: state;
492-
493-
console.log(
494-
chalk.dim(" ") +
495-
statusIcon +
496-
" " +
497-
chalk.dim(scenarioNameTrunc.padEnd(52)) +
498-
statusColor(scoreStr),
499-
);
500-
}
501496
}
502497

503498
console.log();
@@ -547,6 +542,13 @@ export async function watchBenchmarkJob(id: string) {
547542
const SPINNER_INTERVAL_MS = 100;
548543
const UPDATES_PER_POLL = Math.floor(POLL_INTERVAL_MS / SPINNER_INTERVAL_MS);
549544

545+
// Handle terminal resize - clear screen to prevent artifacts
546+
let needsFullRedraw = false;
547+
const handleResize = () => {
548+
needsFullRedraw = true;
549+
};
550+
process.stdout.on("resize", handleResize);
551+
550552
try {
551553
let tick = 0;
552554
let progressList = await fetchAllRunsProgress(job);
@@ -572,6 +574,7 @@ export async function watchBenchmarkJob(id: string) {
572574
chalk.bold.cyan(`Benchmark Job: ${jobName}`) +
573575
chalk.dim(` (${elapsedStr})`),
574576
);
577+
screenLines.push(chalk.dim(`ID: ${job.id}`));
575578
screenLines.push(chalk.dim(`State: ${job.state}`));
576579
screenLines.push("");
577580

@@ -585,6 +588,12 @@ export async function watchBenchmarkJob(id: string) {
585588
screenLines.push("");
586589
screenLines.push(chalk.dim("Press Ctrl+C to exit"));
587590

591+
// Force full clear on resize to prevent artifacts
592+
if (needsFullRedraw) {
593+
process.stdout.write(ANSI.clearScreen);
594+
needsFullRedraw = false;
595+
}
596+
588597
// Render the screen
589598
renderScreen(screenLines);
590599

@@ -601,11 +610,27 @@ export async function watchBenchmarkJob(id: string) {
601610
await sleep(SPINNER_INTERVAL_MS);
602611
}
603612
} finally {
613+
process.stdout.off("resize", handleResize);
604614
cleanup();
605615
}
606616

607-
// Show final results
617+
// Calculate total elapsed time
618+
const totalElapsed = Date.now() - startTime;
619+
const totalElapsedStr = formatDuration(totalElapsed);
620+
621+
// Show completion message
622+
console.log(chalk.green.bold("Benchmark job completed!"));
623+
console.log(chalk.dim(`Total time: ${totalElapsedStr}`));
624+
625+
// Show final results (summary only)
608626
printResultsTable(job);
627+
628+
// Show hint for full results
629+
console.log(
630+
chalk.dim(
631+
`To see full results, run: rli benchmark-job summary -e ${job.id}`,
632+
),
633+
);
609634
} catch (error) {
610635
outputError("Failed to watch benchmark job", error);
611636
}

src/utils/commands.ts

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1054,16 +1054,17 @@ export function createProgram(): Command {
10541054
});
10551055

10561056
benchmarkJob
1057-
.command("status <id>")
1058-
.description("Get benchmark job status and results")
1057+
.command("summary <id>")
1058+
.description("Get benchmark job summary and results")
1059+
.option("-e, --extended", "Show individual scenario results")
10591060
.option(
10601061
"-o, --output [format]",
10611062
"Output format: text|json|yaml (default: text)",
10621063
)
10631064
.action(async (id, options) => {
1064-
const { statusBenchmarkJob } =
1065-
await import("../commands/benchmark-job/status.js");
1066-
await statusBenchmarkJob(id, options);
1065+
const { summaryBenchmarkJob } =
1066+
await import("../commands/benchmark-job/summary.js");
1067+
await summaryBenchmarkJob(id, options);
10671068
});
10681069

10691070
benchmarkJob

0 commit comments

Comments
 (0)