Merge pull request #4351 from nightt5879/nightt5879/issue-4335-provider-aware-scorecard

fix(scorecard): bind costs to provider routes

Author: Hmbown

CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Make offline `scorecard` pricing provider-aware: `turn_end` records carry the
+  effective route and a non-secret billing surface, runtime exports and
+  supported aliases ingest cleanly, legacy/unknown routes remain explicitly
+  unpriced, and route-scoped cache and recorded-time pricing replace model-only
+  guesses. Historical runtime aggregates use each turn's recorded time;
+  costless catalog routes fail closed while exact provider-owned hand-price
+  rows remain available. StepFun PAYG and Step Plan usage now stay distinct
+  without persisting raw endpoint URLs, so subscription quota is never reported
+  as token spend (#4335). Completion-only shell, manual-compaction, and purge
+  events remain visible to `turn_end` observers as explicitly non-model
+  lifecycle records. This builds on the scorecard introduced by @findshan in
+  #3388.
+
 ## [0.8.68] - 2026-07-13
 
 Release-candidate notes for the underwater release: the TUI's default shell is

config.example.toml

@@ -499,12 +499,12 @@ max_subagents = 10 # optional (1-20)
 # # base_url = "https://api.z.ai/api/paas/v4"
 # model = "GLM-5.2"                             # default; GLM-5.1 is the smaller model, GLM-5-Turbo the fast sub-agent sibling
 
-# StepFun / StepFlash direct OpenAI-compatible endpoint (https://platform.stepfun.com)
+# StepFun / StepFlash direct OpenAI-compatible endpoint (https://platform.stepfun.ai)
 [providers.stepfun]
 # api_key = "YOUR_STEPFUN_API_KEY"               # or STEP_API_KEY
 # base_url = "https://api.stepfun.ai/v1"          # or STEP_BASE_URL
 # # Coding Plan endpoint:
-# # base_url = "https://api.stepfun.com/step_plan/v1"
+# # base_url = "https://api.stepfun.ai/step_plan/v1"
 # model = "step-3.7-flash"                       # or STEPFUN_MODEL / STEP_MODEL
 
 # MiniMax direct OpenAI-compatible endpoint (https://platform.minimax.io)

crates/cli/src/lib.rs

@@ -423,6 +423,10 @@ struct LaneArgs {
 }
 
 #[derive(Debug, Subcommand)]
+// Clap constructs this command enum once at process startup. Keeping the
+// fields inline makes the generated CLI shape explicit; boxing them only to
+// reduce this transient value would add indirection without runtime benefit.
+#[allow(clippy::large_enum_variant)]
 enum LaneCommand {
     /// List known lanes (newest first).
     List {

crates/tui/CHANGELOG.md

@@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Make offline `scorecard` pricing provider-aware: `turn_end` records carry the
+  effective route and a non-secret billing surface, runtime exports and
+  supported aliases ingest cleanly, legacy/unknown routes remain explicitly
+  unpriced, and route-scoped cache and recorded-time pricing replace model-only
+  guesses. Historical runtime aggregates use each turn's recorded time;
+  costless catalog routes fail closed while exact provider-owned hand-price
+  rows remain available. StepFun PAYG and Step Plan usage now stay distinct
+  without persisting raw endpoint URLs, so subscription quota is never reported
+  as token spend (#4335). Completion-only shell, manual-compaction, and purge
+  events remain visible to `turn_end` observers as explicitly non-model
+  lifecycle records. This builds on the scorecard introduced by @findshan in
+  #3388.
+
 ## [0.8.68] - 2026-07-13
 
 Release-candidate notes for the underwater release: the TUI's default shell is

crates/tui/src/compaction.rs

@@ -927,6 +927,9 @@ fn truncate_retained_block(label: &str, content: &mut String, max_chars: usize)
     true
 }
 
+// A match guard cannot mutably borrow `content`; keeping the mutation inside
+// the arm updates both retained representations together without indirection.
+#[allow(clippy::collapsible_match)]
 fn sanitize_retained_messages(mut messages: Vec<Message>) -> Vec<Message> {
     for message in &mut messages {
         for block in &mut message.content {

crates/tui/src/core/engine.rs

@@ -66,7 +66,7 @@ use crate::working_set::WorkingSet;
 #[cfg(test)]
 use super::authority::agent_approval_mode_for_turn;
 use super::authority::{TurnAuthority, effective_input_policy, shell_policy_for_mode};
-use super::events::{Event, TurnOutcomeStatus};
+use super::events::{Event, TurnOutcomeStatus, TurnRoute};
 use super::ops::{
     Op, ProviderRuntimeStatus, SessionSnapshot, USER_SHELL_TOOL_ID_PREFIX, UserInputProvenance,
 };
@@ -1184,6 +1184,8 @@ impl Engine {
             .tx_event
             .send(Event::TurnStarted {
                 turn_id: turn_id.clone(),
+                created_at: chrono::Utc::now(),
+                route: None,
             })
             .await;
 
@@ -2466,6 +2468,11 @@ impl Engine {
         // Create turn context first so start event includes a stable turn id.
         let mut turn = TurnContext::new(self.config.max_steps);
         self.turn_counter = self.turn_counter.saturating_add(1);
+        let turn_route = TurnRoute {
+            provider: provider.unwrap_or(self.api_provider),
+            model: model.clone(),
+            auto_model,
+        };
 
         // Emit turn started event IMMEDIATELY so the UI knows the turn is
         // active. The snapshot below can take 30+ seconds on slow filesystems
@@ -2474,6 +2481,8 @@ impl Engine {
             .tx_event
             .send(Event::TurnStarted {
                 turn_id: turn.id.clone(),
+                created_at: chrono::Utc::now(),
+                route: Some(turn_route),
             })
             .await;
 

crates/tui/src/core/engine/tests.rs

@@ -773,12 +773,20 @@ async fn operate_admission_blocks_unready_nontrivial_but_allows_act_and_trivial(
 
     let mut saw_blocker = false;
     let mut saw_operate_complete = false;
+    let mut saw_operate_route = false;
     let mut operate_rx = operate_handle.rx_event.write().await;
     while let Some(event) = tokio::time::timeout(model_turn_event_timeout(), operate_rx.recv())
         .await
         .expect("timed out waiting for Operate blocker")
     {
         match event {
+            Event::TurnStarted { route, .. } => {
+                let route = route.expect("model turn route");
+                assert_eq!(route.provider, ApiProvider::Deepseek);
+                assert_eq!(route.model, crate::config::DEFAULT_TEXT_MODEL);
+                assert!(!route.auto_model);
+                saw_operate_route = true;
+            }
             Event::Error { envelope, .. } => {
                 saw_blocker = true;
                 assert!(
@@ -814,6 +822,10 @@ async fn operate_admission_blocks_unready_nontrivial_but_allows_act_and_trivial(
         saw_blocker,
         "Operate must surface an actionable readiness blocker"
     );
+    assert!(
+        saw_operate_route,
+        "model turns must publish route provenance"
+    );
     assert!(
         saw_operate_complete,
         "blocked Operate turn must terminate cleanly"
@@ -2953,8 +2965,9 @@ async fn run_shell_command_op_requests_approval_and_executes_shell() {
     let mut rx = handle.rx_event.write().await;
     while let Some(event) = rx.recv().await {
         match event {
-            Event::TurnStarted { turn_id } => {
+            Event::TurnStarted { turn_id, route, .. } => {
                 assert!(turn_id.starts_with(USER_SHELL_TOOL_ID_PREFIX));
+                assert!(route.is_none());
             }
             Event::ToolCallStarted { id, name, input } => {
                 saw_started = true;

crates/tui/src/core/events.rs

@@ -5,8 +5,10 @@
 
 use std::{path::PathBuf, sync::Arc};
 
+use chrono::{DateTime, Utc};
 use serde_json::Value;
 
+use crate::config::ApiProvider;
 use crate::error_taxonomy::ErrorEnvelope;
 use crate::models::{Message, SystemPrompt, Tool, Usage};
 use crate::tools::goal::GoalSnapshot;
@@ -22,6 +24,18 @@ pub enum TurnOutcomeStatus {
     Failed,
 }
 
+/// Provider/model route resolved for a model-backed turn.
+///
+/// Carried with `TurnStarted` so hosts can retain provenance until the matching
+/// `TurnComplete` without relying on mutable global selection state. Non-model
+/// turns such as composer `!` shell commands use no route.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct TurnRoute {
+    pub provider: ApiProvider,
+    pub model: String,
+    pub auto_model: bool,
+}
+
 /// Events emitted by the engine to update the UI.
 #[derive(Debug, Clone)]
 pub enum Event {
@@ -81,7 +95,11 @@ pub enum Event {
 
     // === Turn Lifecycle ===
     /// A new turn has started (user sent a message)
-    TurnStarted { turn_id: String },
+    TurnStarted {
+        turn_id: String,
+        created_at: DateTime<Utc>,
+        route: Option<TurnRoute>,
+    },
 
     /// The turn is complete (no more tool calls)
     TurnComplete {

crates/tui/src/cost_status.rs

@@ -36,7 +36,8 @@ fn cell() -> &'static Mutex<CostEstimate> {
 /// cost via [`crate::pricing::calculate_turn_cost_estimate_for_provider`] and
 /// adds it to the pending pool. Cheap; takes a short-lived lock
 /// and returns. No-op when the provider does not expose authoritative runtime
-/// pricing (for example ChatGPT/Codex OAuth) or the model is unknown.
+/// pricing (for example ChatGPT/Codex OAuth), needs route provenance that this
+/// side channel does not carry (StepFun PAYG vs Plan), or the model is unknown.
 pub fn report(provider: ApiProvider, model: &str, usage: &Usage) {
     let Some(cost) =
         crate::pricing::calculate_turn_cost_estimate_for_provider(provider, model, usage)
@@ -127,6 +128,15 @@ mod tests {
         assert_eq!(drain(), CostEstimate::default());
     }
 
+    #[test]
+    fn report_skips_stepfun_without_billing_surface() {
+        let _g = serial_lock();
+        reset_for_tests();
+        report(ApiProvider::Stepfun, "step-3.7-flash", &small_usage());
+        report(ApiProvider::Openrouter, "step-3.7-flash", &small_usage());
+        assert_eq!(drain(), CostEstimate::default());
+    }
+
     #[test]
     fn report_accumulates_across_multiple_calls() {
         let _g = serial_lock();

crates/tui/src/hooks.rs

@@ -14,6 +14,7 @@
 // Note: anyhow is available if needed for future error handling
 #[allow(unused_imports)]
 use anyhow::{Context, Result};
+use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::collections::HashMap;
@@ -613,7 +614,12 @@ pub struct TurnEndTotals {
 /// Input used to build the structured `turn_end` observer payload.
 pub struct TurnEndPayloadInput<'a> {
     pub context: &'a HookContext,
-    pub turn_id: Option<&'a str>,
+    pub created_at: DateTime<Utc>,
+    pub model_backed: bool,
+    pub provider: Option<&'a str>,
+    pub billing_surface: Option<&'a str>,
+    pub model: Option<&'a str>,
+    pub turn_id: &'a str,
     pub status: &'a str,
     pub error: Option<&'a str>,
     pub duration: Duration,
@@ -1291,7 +1297,11 @@ pub fn turn_end_payload(input: TurnEndPayloadInput<'_>) -> serde_json::Value {
         "session_id": input.context.session_id.as_deref(),
         "workspace": input.context.workspace.as_ref().map(|path| path.display().to_string()),
         "mode": input.context.mode.as_deref(),
-        "model": input.context.model.as_deref(),
+        "created_at": input.created_at.to_rfc3339(),
+        "model_backed": input.model_backed,
+        "provider": input.provider,
+        "billing_surface": input.billing_surface,
+        "model": input.model.or(input.context.model.as_deref()),
         "turn_id": input.turn_id,
         "status": input.status,
         "error": input.error,
@@ -1576,7 +1586,12 @@ NOEQUAL line dropped
 
         let payload = super::turn_end_payload(TurnEndPayloadInput {
             context: &context,
-            turn_id: Some("turn_123"),
+            created_at: "2026-07-12T10:30:00Z".parse().expect("timestamp"),
+            model_backed: true,
+            provider: Some("deepseek"),
+            billing_surface: Some("test-payg"),
+            model: Some("deepseek-v4-pro"),
+            turn_id: "turn_123",
             status: "completed",
             error: None,
             duration: Duration::from_millis(321),
@@ -1595,7 +1610,12 @@ NOEQUAL line dropped
         assert_eq!(payload["session_id"], "sess_test");
         assert_eq!(payload["workspace"], "/tmp/codewhale");
         assert_eq!(payload["mode"], "agent");
-        assert_eq!(payload["model"], "deepseek-v4");
+        assert_eq!(payload["created_at"], "2026-07-12T10:30:00+00:00");
+        assert_eq!(payload["model_backed"], true);
+        assert_eq!(payload["provider"], "deepseek");
+        assert_eq!(payload["billing_surface"], "test-payg");
+        assert!(payload.get("base_url").is_none());
+        assert_eq!(payload["model"], "deepseek-v4-pro");
         assert_eq!(payload["turn_id"], "turn_123");
         assert_eq!(payload["status"], "completed");
         assert_eq!(payload["error"], serde_json::Value::Null);
@@ -1883,7 +1903,12 @@ printf '%s\n' '{{"text":"stdout is not a mutation contract"}}'
         let context = submit_context(&dir).with_tokens(15);
         let payload = super::turn_end_payload(TurnEndPayloadInput {
             context: &context,
-            turn_id: Some("turn_observed"),
+            created_at: "2026-07-12T10:30:00Z".parse().expect("timestamp"),
+            model_backed: true,
+            provider: Some("openai"),
+            billing_surface: None,
+            model: Some("gpt-5.5"),
+            turn_id: "turn_observed",
             status: "completed",
             error: None,
             duration: Duration::from_millis(7),
@@ -1912,6 +1937,9 @@ printf '%s\n' '{{"text":"stdout is not a mutation contract"}}'
             serde_json::from_str(&std::fs::read_to_string(out).expect("payload written"))
                 .expect("valid JSON payload");
         assert_eq!(captured["event"], "turn_end");
+        assert_eq!(captured["created_at"], "2026-07-12T10:30:00+00:00");
+        assert_eq!(captured["provider"], "openai");
+        assert_eq!(captured["model"], "gpt-5.5");
         assert_eq!(captured["turn_id"], "turn_observed");
         assert_eq!(captured["totals"]["input_tokens"], 12);
         assert_eq!(captured["totals"]["output_tokens"], 3);