Skip to content

Commit 2abe85c

Browse files
jrvb-rljrvb-rl
authored
feat: add benchmark list features (#154)
1 parent 32f4f72 commit 2abe85c

3 files changed

Lines changed: 290 additions & 0 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,7 @@ rli mcp install # Install Runloop MCP server configurat
187187
rli benchmark-job run # Run a benchmark job with one or more ...
188188
rli benchmark-job summary <id> # Get benchmark job summary and results
189189
rli benchmark-job watch <id> # Watch benchmark job progress in real-...
190+
rli benchmark-job list # List benchmark jobs
190191
```
191192

192193

‎src/commands/benchmark-job/list.ts‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
/**
2+
* List benchmark jobs command
3+
*/
4+
5+
import chalk from "chalk";
6+
import {
7+
listBenchmarkJobs,
8+
type BenchmarkJob,
9+
} from "../../services/benchmarkJobService.js";
10+
import { output, outputError } from "../../utils/output.js";
11+
12+
interface ListOptions {
13+
days?: string;
14+
all?: boolean;
15+
status?: string;
16+
output?: string;
17+
}
18+
19+
const VALID_STATES = [
20+
"initializing",
21+
"queued",
22+
"running",
23+
"completed",
24+
"failed",
25+
"cancelled",
26+
"timeout",
27+
];
28+
29+
const PAGE_SIZE = 100;
30+
31+
// --- Time formatting ---
32+
33+
function formatTimeAgo(timestampMs: number): string {
34+
const diffMs = Date.now() - timestampMs;
35+
const diffMinutes = Math.floor(diffMs / 60_000);
36+
const diffHours = Math.floor(diffMs / 3_600_000);
37+
const diffDays = Math.floor(diffMs / 86_400_000);
38+
39+
if (diffMinutes < 1) return "just now";
40+
if (diffMinutes < 60) return `${diffMinutes}m ago`;
41+
if (diffHours < 24) return `${diffHours}h ago`;
42+
if (diffDays < 7) return `${diffDays}d ago`;
43+
44+
const date = new Date(timestampMs);
45+
return date.toLocaleDateString("en-US", { month: "short", day: "numeric" });
46+
}
47+
48+
// --- Job stats aggregation ---
49+
50+
interface JobStats {
51+
done: number;
52+
total: number;
53+
errors: number;
54+
avgScore: number | null;
55+
}
56+
57+
function aggregateJobStats(job: BenchmarkJob): JobStats {
58+
const outcomes = job.benchmark_outcomes || [];
59+
const scenarioCount = job.job_spec?.scenario_ids?.length || 0;
60+
const agentCount = job.job_spec?.agent_configs?.length || 1;
61+
const total = scenarioCount * agentCount;
62+
63+
let done = 0;
64+
let errors = 0;
65+
let scoreSum = 0;
66+
let scoreCount = 0;
67+
68+
for (const outcome of outcomes) {
69+
done += outcome.n_completed + outcome.n_failed + outcome.n_timeout;
70+
errors += outcome.n_failed + outcome.n_timeout;
71+
if (outcome.average_score !== undefined && outcome.average_score !== null) {
72+
scoreSum += outcome.average_score;
73+
scoreCount++;
74+
}
75+
}
76+
77+
return {
78+
done,
79+
total: total || done,
80+
errors,
81+
avgScore: scoreCount > 0 ? scoreSum / scoreCount : null,
82+
};
83+
}
84+
85+
// --- Status coloring ---
86+
87+
function colorState(state: string): string {
88+
switch (state) {
89+
case "running":
90+
return chalk.yellow(state);
91+
case "completed":
92+
return chalk.green(state);
93+
case "failed":
94+
case "timeout":
95+
return chalk.red(state);
96+
case "cancelled":
97+
return chalk.dim(state);
98+
case "initializing":
99+
case "queued":
100+
return chalk.cyan(state);
101+
default:
102+
return state;
103+
}
104+
}
105+
106+
// --- Table printing ---
107+
108+
// Fixed column widths (excluding NAME which is dynamic)
109+
const COL_ID = 16;
110+
const COL_STARTED = 10;
111+
const COL_STATUS = 14;
112+
const COL_DONE = 9;
113+
const COL_ERRORS = 8;
114+
const COL_SCORE = 7;
115+
const FIXED_WIDTH =
116+
COL_ID + COL_STARTED + COL_STATUS + COL_DONE + COL_ERRORS + COL_SCORE + 6; // 6 for spacing
117+
118+
function truncate(str: string, maxLen: number): string {
119+
if (str.length <= maxLen) return str;
120+
return str.slice(0, maxLen - 1) + "…";
121+
}
122+
123+
function printTable(jobs: BenchmarkJob[]): void {
124+
if (jobs.length === 0) {
125+
console.log(chalk.dim("No benchmark jobs found"));
126+
return;
127+
}
128+
129+
const termWidth = process.stdout.columns || 120;
130+
const nameWidth = Math.max(10, termWidth - FIXED_WIDTH);
131+
132+
// Header
133+
const header =
134+
"ID".padEnd(COL_ID) +
135+
" " +
136+
"NAME".padEnd(nameWidth) +
137+
" " +
138+
"STARTED".padEnd(COL_STARTED) +
139+
" " +
140+
"STATUS".padEnd(COL_STATUS) +
141+
" " +
142+
"DONE".padStart(COL_DONE) +
143+
" " +
144+
"ERRORS".padStart(COL_ERRORS) +
145+
" " +
146+
"SCORE".padStart(COL_SCORE);
147+
console.log(chalk.bold(header));
148+
console.log(chalk.dim("─".repeat(Math.min(header.length, termWidth))));
149+
150+
// Rows
151+
for (const job of jobs) {
152+
const stats = aggregateJobStats(job);
153+
154+
const id = truncate(job.id, COL_ID).padEnd(COL_ID);
155+
const name = truncate(job.name || "", nameWidth).padEnd(nameWidth);
156+
const started = formatTimeAgo(job.create_time_ms).padEnd(COL_STARTED);
157+
const status = colorState(job.state || "unknown");
158+
// Pad status accounting for chalk invisible chars
159+
const statusRaw = job.state || "unknown";
160+
const statusPad = " ".repeat(Math.max(0, COL_STATUS - statusRaw.length));
161+
162+
const doneStr = `${stats.done}/${stats.total}`.padStart(COL_DONE);
163+
const errorsStr = String(stats.errors).padStart(COL_ERRORS);
164+
const coloredErrors =
165+
stats.errors > 0 ? chalk.red(errorsStr) : chalk.dim(errorsStr);
166+
167+
let scoreStr: string;
168+
if (stats.avgScore !== null) {
169+
const pct = Math.round(stats.avgScore * 100);
170+
const pctStr = `${pct}%`.padStart(COL_SCORE);
171+
scoreStr = pct >= 50 ? chalk.green(pctStr) : chalk.yellow(pctStr);
172+
} else {
173+
scoreStr = chalk.dim("N/A".padStart(COL_SCORE));
174+
}
175+
176+
console.log(
177+
`${id} ${name} ${started} ${status}${statusPad} ${doneStr} ${coloredErrors} ${scoreStr}`,
178+
);
179+
}
180+
181+
console.log();
182+
console.log(chalk.dim(`${jobs.length} job${jobs.length !== 1 ? "s" : ""}`));
183+
}
184+
185+
// --- Pagination and filtering ---
186+
187+
async function fetchJobs(
188+
cutoffMs: number | null,
189+
statusFilter: Set<string> | null,
190+
): Promise<BenchmarkJob[]> {
191+
const allJobs: BenchmarkJob[] = [];
192+
let cursor: string | undefined;
193+
194+
while (true) {
195+
const result = await listBenchmarkJobs({
196+
limit: PAGE_SIZE,
197+
startingAfter: cursor,
198+
});
199+
200+
for (const job of result.jobs) {
201+
// Stop pagination if we've passed the time cutoff (API returns newest-first)
202+
if (cutoffMs !== null && job.create_time_ms < cutoffMs) {
203+
return applyStatusFilter(allJobs, statusFilter);
204+
}
205+
allJobs.push(job);
206+
}
207+
208+
if (!result.hasMore || result.jobs.length === 0) break;
209+
cursor = result.jobs[result.jobs.length - 1].id;
210+
}
211+
212+
return applyStatusFilter(allJobs, statusFilter);
213+
}
214+
215+
function applyStatusFilter(
216+
jobs: BenchmarkJob[],
217+
statusFilter: Set<string> | null,
218+
): BenchmarkJob[] {
219+
if (!statusFilter) return jobs;
220+
return jobs.filter((job) => statusFilter.has(job.state?.toLowerCase() || ""));
221+
}
222+
223+
// --- Command entry point ---
224+
225+
export async function listBenchmarkJobsCommand(
226+
options: ListOptions,
227+
): Promise<void> {
228+
try {
229+
// Parse status filter
230+
let statusFilter: Set<string> | null = null;
231+
if (options.status) {
232+
const statuses = options.status
233+
.split(",")
234+
.map((s) => s.trim().toLowerCase());
235+
const invalid = statuses.filter((s) => !VALID_STATES.includes(s));
236+
if (invalid.length > 0) {
237+
outputError(
238+
`Invalid status: ${invalid.join(", ")}. Valid: ${VALID_STATES.join(", ")}`,
239+
);
240+
}
241+
statusFilter = new Set(statuses);
242+
}
243+
244+
// Compute time cutoff
245+
let cutoffMs: number | null = null;
246+
if (!options.all) {
247+
const days = options.days ? parseInt(options.days, 10) : 1;
248+
if (isNaN(days) || days <= 0) {
249+
outputError("--days must be a positive integer");
250+
}
251+
cutoffMs = Date.now() - days * 86_400_000;
252+
}
253+
254+
// Fetch and filter
255+
const jobs = await fetchJobs(cutoffMs, statusFilter);
256+
257+
// Sort ascending by create_time_ms (oldest first, most recent at bottom)
258+
jobs.sort((a, b) => a.create_time_ms - b.create_time_ms);
259+
260+
// Output
261+
const format = options.output || "text";
262+
if (format !== "text") {
263+
output(jobs, { format, defaultFormat: "json" });
264+
} else {
265+
printTable(jobs);
266+
}
267+
} catch (error) {
268+
outputError("Failed to list benchmark jobs", error);
269+
}
270+
}

‎src/utils/commands.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,6 +1076,25 @@ export function createProgram(): Command {
10761076
await watchBenchmarkJob(id);
10771077
});
10781078

1079+
benchmarkJob
1080+
.command("list")
1081+
.description("List benchmark jobs")
1082+
.option("--days <n>", "Show jobs from the last N days (default: 1)")
1083+
.option("--all", "Show all jobs (no time filter)")
1084+
.option(
1085+
"--status <statuses>",
1086+
"Filter by status (comma-separated). Valid: initializing, queued, running, completed, failed, cancelled, timeout",
1087+
)
1088+
.option(
1089+
"-o, --output [format]",
1090+
"Output format: text|json|yaml (default: text)",
1091+
)
1092+
.action(async (options) => {
1093+
const { listBenchmarkJobsCommand } =
1094+
await import("../commands/benchmark-job/list.js");
1095+
await listBenchmarkJobsCommand(options);
1096+
});
1097+
10791098
// Hidden command: 'rli mcp' without subcommand starts the server (for Claude Desktop config compatibility)
10801099
program
10811100
.command("mcp-server", { hidden: true })

0 commit comments

Comments
 (0)