Skip to content

Commit cba70ce

Browse files
committed
fix(scorecard): handle sparse runtime records
Deserialize runtime turns before usage is available and filter them from model-backed scoring. Compare complete CNY totals alongside USD totals so cost regressions cannot bypass the threshold gate. Signed-off-by: Nightt <87569709+nightt5879@users.noreply.github.com>
1 parent 67d84e2 commit cba70ce

1 file changed

Lines changed: 64 additions & 17 deletions

File tree

crates/tui/src/scorecard.rs

Lines changed: 64 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -144,15 +144,16 @@ pub struct RecordedTurn {
144144
pub provider: Option<String>,
145145
#[serde(default, alias = "effective_billing_surface")]
146146
pub billing_surface: Option<String>,
147-
#[serde(alias = "effective_model")]
147+
#[serde(default, alias = "effective_model")]
148148
pub model: String,
149-
pub usage: Usage,
149+
#[serde(default)]
150+
pub usage: Option<Usage>,
150151
}
151152

152153
impl RecordedTurn {
153154
#[must_use]
154155
pub fn contributes_to_scorecard(&self) -> bool {
155-
self.model_backed.unwrap_or(true)
156+
self.model_backed.unwrap_or(true) && self.usage.is_some() && !self.model.trim().is_empty()
156157
}
157158
}
158159

@@ -221,19 +222,20 @@ impl Scorecard {
221222
/// while excluding explicitly non-model lifecycle rows.
222223
#[must_use]
223224
pub fn from_recorded_turns(turns: &[RecordedTurn]) -> Self {
224-
Self::from_turn_refs(
225-
turns
226-
.iter()
227-
.filter(|turn| turn.contributes_to_scorecard())
228-
.map(|turn| ScorecardTurnRef {
229-
turn_id: &turn.turn_id,
230-
created_at: turn.created_at.as_ref(),
231-
provider: turn.provider.as_deref(),
232-
billing_surface: turn.billing_surface.as_deref(),
233-
model: &turn.model,
234-
usage: &turn.usage,
235-
}),
236-
)
225+
Self::from_turn_refs(turns.iter().filter_map(|turn| {
226+
if !turn.contributes_to_scorecard() {
227+
return None;
228+
}
229+
let usage = turn.usage.as_ref()?;
230+
Some(ScorecardTurnRef {
231+
turn_id: &turn.turn_id,
232+
created_at: turn.created_at.as_ref(),
233+
provider: turn.provider.as_deref(),
234+
billing_surface: turn.billing_surface.as_deref(),
235+
model: &turn.model,
236+
usage,
237+
})
238+
}))
237239
}
238240

239241
fn from_turn_refs<'a>(turns: impl IntoIterator<Item = ScorecardTurnRef<'a>>) -> Self {
@@ -407,6 +409,14 @@ impl ScorecardMetrics {
407409
current: 0.0,
408410
pct_increase: 100.0,
409411
});
412+
} else if self.cny_cost_complete && baseline.cny_cost_complete {
413+
push_regression(
414+
&mut out,
415+
"total_cost_cny",
416+
baseline.total_cost_cny,
417+
self.total_cost_cny,
418+
threshold_pct,
419+
);
410420
}
411421
push_regression(
412422
&mut out,
@@ -948,7 +958,7 @@ mod tests {
948958
provider: Some(provider.to_string()),
949959
billing_surface: billing_surface.map(str::to_string),
950960
model: model.to_string(),
951-
usage: u.clone(),
961+
usage: Some(u.clone()),
952962
};
953963
let turns = [
954964
recorded(
@@ -1053,6 +1063,27 @@ mod tests {
10531063
assert!(recorded.contributes_to_scorecard());
10541064
}
10551065

1066+
#[test]
1067+
fn runtime_turn_without_usage_is_readable_and_filtered() {
1068+
let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
1069+
"schema_version": 1,
1070+
"id": "queued-runtime-turn",
1071+
"thread_id": "thread-1",
1072+
"status": "queued",
1073+
"input_summary": "waiting to run",
1074+
"created_at": "2026-07-12T10:30:00Z",
1075+
"effective_provider": "openai",
1076+
"effective_model": "gpt-5.5"
1077+
}))
1078+
.expect("parse runtime row before usage is recorded");
1079+
1080+
assert!(recorded.usage.is_none());
1081+
assert!(!recorded.contributes_to_scorecard());
1082+
let card = Scorecard::from_recorded_turns(&[recorded]);
1083+
assert_eq!(card.metrics.turns, 0);
1084+
assert!(card.per_turn.is_empty());
1085+
}
1086+
10561087
#[test]
10571088
fn recorded_non_model_hook_turn_is_excluded_from_model_scorecard() {
10581089
let recorded: RecordedTurn = serde_json::from_value(serde_json::json!({
@@ -1181,6 +1212,22 @@ mod tests {
11811212
);
11821213
}
11831214

1215+
#[test]
1216+
fn regression_flags_complete_cny_cost_increase() {
1217+
let baseline = ScorecardMetrics {
1218+
cny_cost_complete: true,
1219+
total_cost_cny: 0.70,
1220+
..Default::default()
1221+
};
1222+
let current = ScorecardMetrics {
1223+
total_cost_cny: 1.40,
1224+
..baseline.clone()
1225+
};
1226+
1227+
let regs = current.regressions_against(&baseline, 5.0);
1228+
assert!(regs.iter().any(|r| r.metric == "total_cost_cny"));
1229+
}
1230+
11841231
#[test]
11851232
fn legacy_baseline_is_readable_but_cost_is_not_comparable() {
11861233
let baseline: ScorecardMetrics = serde_json::from_value(serde_json::json!({

0 commit comments

Comments
 (0)