Skip to content

Commit 78f8f55

Browse files
authored
fix: scenarios sometimes listed as in progress after bmj completes (#174)
## Description - Refactored the progress bar to sit in a common location - Perform a final round of reconciliation after bmj completes to avoid listing some scenarios as being stuck in "in progress" even though they already finished running. ## Type of Change <!-- Mark the relevant option with an 'x' --> - [X] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Documentation update - [ ] Code refactoring - [ ] Performance improvement - [ ] Test updates ## Related Issues <!-- Link to related issues using #issue-number --> Closes # ## Changes Made <!-- Describe the changes in detail --> ## Testing <!-- Describe how you tested your changes --> - [X] I have tested locally (but this is hard to reproduce) - [X] I have added/updated tests - [X] All existing tests pass ## Checklist - [ ] My code follows the code style of this project - [ ] I have performed a self-review of my own code - [ ] I have commented my code, particularly in hard-to-understand areas - [ ] I have updated the documentation accordingly - [ ] My changes generate no new warnings - [ ] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [ ] Any dependent changes have been merged and published ## Screenshots (if applicable) <!-- Add screenshots to help explain your changes --> ## Additional Notes <!-- Any additional information that reviewers should know -->
1 parent a5828a2 commit 78f8f55

5 files changed

Lines changed: 603 additions & 331 deletions

File tree

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
/**
2+
* Shared benchmark job progress helpers
3+
*
4+
* Provides terminal-state classification and run reconciliation logic
5+
* used by watch, summary, and detail views.
6+
*/
7+
8+
import type {
9+
BenchmarkJob,
10+
ScenarioRun,
11+
} from "../../services/benchmarkJobService.js";
12+
13+
// ---------------------------------------------------------------------------
14+
// State Classification
15+
// ---------------------------------------------------------------------------
16+
17+
/** Job states that indicate completion */
18+
export const JOB_COMPLETED_STATES = [
19+
"completed",
20+
"failed",
21+
"canceled",
22+
"cancelled",
23+
"timeout",
24+
];
25+
26+
/** Scenario run states that count as finished (no longer running) */
27+
export const SCENARIO_COMPLETED_STATES = [
28+
"completed",
29+
"failed",
30+
"canceled",
31+
"cancelled",
32+
"timeout",
33+
"error",
34+
"scored", // treat scored as complete, consistent with run detail screen
35+
];
36+
37+
/** Check if a job is in a terminal state */
38+
export function isJobCompleted(state: string | undefined | null): boolean {
39+
if (!state) return false;
40+
return JOB_COMPLETED_STATES.includes(state.toLowerCase());
41+
}
42+
43+
/** Check if a scenario is in a terminal state */
44+
export function isScenarioCompleted(state: string | undefined | null): boolean {
45+
if (!state) return false;
46+
return SCENARIO_COMPLETED_STATES.includes(state.toLowerCase());
47+
}
48+
49+
// ---------------------------------------------------------------------------
50+
// Progress Stats
51+
// ---------------------------------------------------------------------------
52+
53+
/** In-progress scenario info for display */
54+
export interface InProgressScenario {
55+
name: string;
56+
state: string;
57+
startTimeMs?: number;
58+
}
59+
60+
/** Progress stats for a benchmark run */
61+
export interface RunProgress {
62+
benchmarkRunId: string;
63+
agentName: string;
64+
modelName?: string;
65+
state: string;
66+
expectedTotal: number;
67+
started: number;
68+
running: number;
69+
scoring: number;
70+
finished: number;
71+
avgScore: number | null;
72+
inProgressScenarios: InProgressScenario[];
73+
}
74+
75+
// ---------------------------------------------------------------------------
76+
// Agent Info Extraction
77+
// ---------------------------------------------------------------------------
78+
79+
type InProgressRun = NonNullable<BenchmarkJob["in_progress_runs"]>[number];
80+
81+
/** Get agent info from in_progress_run */
82+
export function getAgentInfo(run: InProgressRun): {
83+
name: string;
84+
model?: string;
85+
} {
86+
const agentConfig = run.agent_config;
87+
if (agentConfig && agentConfig.type === "job_agent") {
88+
return {
89+
name: agentConfig.name,
90+
model: agentConfig.model_name ?? undefined,
91+
};
92+
}
93+
return { name: "unknown" };
94+
}
95+
96+
// ---------------------------------------------------------------------------
97+
// Progress Calculation
98+
// ---------------------------------------------------------------------------
99+
100+
/** Calculate progress from scenario runs */
101+
export function calculateRunProgress(
102+
benchmarkRunId: string,
103+
agentName: string,
104+
modelName: string | undefined,
105+
state: string,
106+
expectedTotal: number,
107+
scenarioRuns: ScenarioRun[],
108+
): RunProgress {
109+
let running = 0;
110+
let scoring = 0;
111+
let finished = 0;
112+
let scoreSum = 0;
113+
let scoreCount = 0;
114+
const inProgressScenarios: InProgressScenario[] = [];
115+
116+
for (const scenario of scenarioRuns) {
117+
const scenarioState = scenario.state?.toLowerCase() || "";
118+
119+
if (isScenarioCompleted(scenarioState)) {
120+
finished++;
121+
const score = scenario.scoring_contract_result?.score;
122+
if (score !== undefined && score !== null) {
123+
scoreSum += score;
124+
scoreCount++;
125+
}
126+
} else if (scenarioState === "scoring") {
127+
scoring++;
128+
inProgressScenarios.push({
129+
name: scenario.name || scenario.scenario_id || "unknown",
130+
state: scenarioState,
131+
startTimeMs: scenario.start_time_ms,
132+
});
133+
} else if (scenarioState === "running") {
134+
running++;
135+
inProgressScenarios.push({
136+
name: scenario.name || scenario.scenario_id || "unknown",
137+
state: scenarioState,
138+
startTimeMs: scenario.start_time_ms,
139+
});
140+
} else if (scenarioState && scenarioState !== "pending") {
141+
inProgressScenarios.push({
142+
name: scenario.name || scenario.scenario_id || "unknown",
143+
state: scenarioState,
144+
startTimeMs: scenario.start_time_ms,
145+
});
146+
}
147+
}
148+
149+
return {
150+
benchmarkRunId,
151+
agentName,
152+
modelName,
153+
state,
154+
expectedTotal,
155+
started: scenarioRuns.length,
156+
running,
157+
scoring,
158+
finished,
159+
avgScore: scoreCount > 0 ? scoreSum / scoreCount : null,
160+
inProgressScenarios,
161+
};
162+
}
163+
164+
// ---------------------------------------------------------------------------
165+
// Run Reconciliation
166+
// ---------------------------------------------------------------------------
167+
168+
/**
169+
* Build progress for all runs in a job, preferring completed outcomes over
170+
* stale in-progress data for the same benchmark_run_id.
171+
*
172+
* @param job The benchmark job
173+
* @param fetchScenarioRuns Callback to fetch scenario runs for a benchmark run
174+
*/
175+
export async function fetchAllRunsProgress(
176+
job: BenchmarkJob,
177+
fetchScenarioRuns: (benchmarkRunId: string) => Promise<ScenarioRun[]>,
178+
): Promise<RunProgress[]> {
179+
const results: RunProgress[] = [];
180+
181+
// Get expected scenario count from job spec
182+
const expectedTotal = job.job_spec?.scenario_ids?.length || 0;
183+
184+
// Track which runs we've already added from completed outcomes
185+
const completedRunIds = new Set<string>();
186+
187+
// First, add completed runs from benchmark_outcomes (authoritative)
188+
const completedOutcomes = job.benchmark_outcomes || [];
189+
for (const outcome of completedOutcomes) {
190+
completedRunIds.add(outcome.benchmark_run_id);
191+
192+
const scenarioOutcomes = outcome.scenario_outcomes || [];
193+
let scoreSum = 0;
194+
let scoreCount = 0;
195+
for (const s of scenarioOutcomes) {
196+
if (s.score !== undefined && s.score !== null) {
197+
scoreSum += s.score;
198+
scoreCount++;
199+
}
200+
}
201+
results.push({
202+
benchmarkRunId: outcome.benchmark_run_id,
203+
agentName: outcome.agent_name,
204+
modelName: outcome.model_name ?? undefined,
205+
state: "completed",
206+
expectedTotal: expectedTotal || scenarioOutcomes.length,
207+
started: scenarioOutcomes.length,
208+
running: 0,
209+
scoring: 0,
210+
finished: scenarioOutcomes.length,
211+
avgScore: scoreCount > 0 ? scoreSum / scoreCount : null,
212+
inProgressScenarios: [],
213+
});
214+
}
215+
216+
// Then, fetch progress for in-progress runs that are NOT already in outcomes
217+
const inProgressRuns = job.in_progress_runs || [];
218+
const progressPromises = inProgressRuns
219+
.filter((run) => !completedRunIds.has(run.benchmark_run_id))
220+
.map(async (run) => {
221+
const agentInfo = getAgentInfo(run);
222+
const scenarioRuns = await fetchScenarioRuns(run.benchmark_run_id);
223+
return calculateRunProgress(
224+
run.benchmark_run_id,
225+
agentInfo.name,
226+
agentInfo.model,
227+
run.state,
228+
expectedTotal,
229+
scenarioRuns,
230+
);
231+
});
232+
233+
const inProgressResults = await Promise.all(progressPromises);
234+
results.push(...inProgressResults);
235+
236+
return results;
237+
}

0 commit comments

Comments
 (0)