|
| 1 | +//! Agent statistics extracted from Copilot CLI OpenTelemetry output. |
| 2 | +//! |
| 3 | +//! The Copilot CLI can write OTel spans and metrics to a JSONL file via |
| 4 | +//! `COPILOT_OTEL_FILE_EXPORTER_PATH`. This module parses that file to |
| 5 | +//! extract agent execution statistics (token usage, duration, model, |
| 6 | +//! tool calls, turns) for inclusion in safe output write actions. |
| 7 | +
|
| 8 | +use anyhow::{Context, Result}; |
| 9 | +use serde_json::Value; |
| 10 | +use std::path::Path; |
| 11 | + |
| 12 | +/// Agent execution statistics parsed from OTel JSONL. |
| 13 | +#[derive(Debug, Clone)] |
| 14 | +pub struct AgentStats { |
| 15 | + /// Agent name from front matter. |
| 16 | + pub agent_name: String, |
| 17 | + /// AI model used (e.g., "claude-sonnet-4.5"). |
| 18 | + pub model: Option<String>, |
| 19 | + /// Total input tokens across all LLM calls. |
| 20 | + pub input_tokens: u64, |
| 21 | + /// Total output tokens across all LLM calls. |
| 22 | + pub output_tokens: u64, |
| 23 | + /// Wall-clock duration in seconds. |
| 24 | + pub duration_seconds: f64, |
| 25 | + /// Number of tool invocations. |
| 26 | + pub tool_calls: u64, |
| 27 | + /// Number of LLM round-trips (turns). |
| 28 | + pub turns: u64, |
| 29 | +} |
| 30 | + |
| 31 | +/// OTel JSONL filename written by Copilot CLI. |
| 32 | +pub const OTEL_FILENAME: &str = "otel.jsonl"; |
| 33 | + |
| 34 | +impl AgentStats { |
| 35 | + /// Parse agent stats from an OTel JSONL file. |
| 36 | + /// |
| 37 | + /// Uses [`crate::ndjson::read_ndjson_file`] for file I/O, then |
| 38 | + /// extracts stats from the parsed entries. |
| 39 | + pub async fn from_otel_file(path: &Path, agent_name: &str) -> Result<Self> { |
| 40 | + let entries = crate::ndjson::read_ndjson_file(path) |
| 41 | + .await |
| 42 | + .with_context(|| format!("Failed to read OTel file: {}", path.display()))?; |
| 43 | + Self::from_otel_entries(&entries, agent_name) |
| 44 | + } |
| 45 | + |
| 46 | + /// Extract stats from pre-parsed OTel JSONL entries. |
| 47 | + /// |
| 48 | + /// Looks for: |
| 49 | + /// - The last `invoke_agent` span for aggregated tokens, model, turns, duration |
| 50 | + /// - `execute_tool` spans for tool call count |
| 51 | + pub fn from_otel_entries(entries: &[Value], agent_name: &str) -> Result<Self> { |
| 52 | + let mut stats = AgentStats { |
| 53 | + agent_name: agent_name.to_string(), |
| 54 | + model: None, |
| 55 | + input_tokens: 0, |
| 56 | + output_tokens: 0, |
| 57 | + duration_seconds: 0.0, |
| 58 | + tool_calls: 0, |
| 59 | + turns: 0, |
| 60 | + }; |
| 61 | + |
| 62 | + // Find the last invoke_agent span (contains aggregated totals) |
| 63 | + let last_agent_span = entries |
| 64 | + .iter() |
| 65 | + .filter(|e| { |
| 66 | + e.get("type").and_then(|t| t.as_str()) == Some("span") |
| 67 | + && e.get("name").and_then(|n| n.as_str()) == Some("invoke_agent") |
| 68 | + }) |
| 69 | + .last(); |
| 70 | + |
| 71 | + if let Some(span) = last_agent_span { |
| 72 | + let attrs = span.get("attributes").cloned().unwrap_or(Value::Null); |
| 73 | + |
| 74 | + // Model |
| 75 | + stats.model = attrs |
| 76 | + .get("gen_ai.request.model") |
| 77 | + .and_then(|v| v.as_str()) |
| 78 | + .map(|s| s.to_string()); |
| 79 | + |
| 80 | + // Tokens (aggregated across all chat spans) |
| 81 | + stats.input_tokens = attrs |
| 82 | + .get("gen_ai.usage.input_tokens") |
| 83 | + .and_then(|v| v.as_u64()) |
| 84 | + .unwrap_or(0); |
| 85 | + stats.output_tokens = attrs |
| 86 | + .get("gen_ai.usage.output_tokens") |
| 87 | + .and_then(|v| v.as_u64()) |
| 88 | + .unwrap_or(0); |
| 89 | + |
| 90 | + // Turns |
| 91 | + stats.turns = attrs |
| 92 | + .get("github.copilot.turn_count") |
| 93 | + .and_then(|v| v.as_u64()) |
| 94 | + .unwrap_or(0); |
| 95 | + |
| 96 | + // Duration from startTime/endTime ([seconds, nanoseconds] arrays) |
| 97 | + stats.duration_seconds = compute_duration(span); |
| 98 | + } |
| 99 | + |
| 100 | + // Count execute_tool spans |
| 101 | + stats.tool_calls = entries |
| 102 | + .iter() |
| 103 | + .filter(|e| { |
| 104 | + e.get("type").and_then(|t| t.as_str()) == Some("span") |
| 105 | + && e.get("name") |
| 106 | + .and_then(|n| n.as_str()) |
| 107 | + .is_some_and(|n| n.starts_with("execute_tool")) |
| 108 | + }) |
| 109 | + .count() as u64; |
| 110 | + |
| 111 | + Ok(stats) |
| 112 | + } |
| 113 | + |
| 114 | + /// Total tokens (input + output). |
| 115 | + pub fn total_tokens(&self) -> u64 { |
| 116 | + self.input_tokens + self.output_tokens |
| 117 | + } |
| 118 | + |
| 119 | + /// Render as a collapsible markdown stats block. |
| 120 | + pub fn to_markdown(&self) -> String { |
| 121 | + let duration = format_duration(self.duration_seconds); |
| 122 | + let model = self.model.as_deref().unwrap_or("unknown"); |
| 123 | + |
| 124 | + format!( |
| 125 | + "\n---\n\ |
| 126 | + <details>\n\ |
| 127 | + <summary>\u{1F916} Agent Stats ({name})</summary>\n\ |
| 128 | + \n\ |
| 129 | + | Metric | Value |\n\ |
| 130 | + |--------|-------|\n\ |
| 131 | + | Model | {model} |\n\ |
| 132 | + | Tokens | {input} input / {output} output ({total} total) |\n\ |
| 133 | + | Duration | {duration} |\n\ |
| 134 | + | Tool calls | {tools} |\n\ |
| 135 | + | Turns | {turns} |\n\ |
| 136 | + \n\ |
| 137 | + </details>\n", |
| 138 | + name = self.agent_name, |
| 139 | + model = model, |
| 140 | + input = format_number(self.input_tokens), |
| 141 | + output = format_number(self.output_tokens), |
| 142 | + total = format_number(self.total_tokens()), |
| 143 | + duration = duration, |
| 144 | + tools = self.tool_calls, |
| 145 | + turns = self.turns, |
| 146 | + ) |
| 147 | + } |
| 148 | +} |
| 149 | + |
| 150 | +/// Compute duration from OTel span startTime/endTime. |
| 151 | +/// |
| 152 | +/// Times are `[seconds, nanoseconds]` arrays. |
| 153 | +fn compute_duration(span: &Value) -> f64 { |
| 154 | + let start = parse_otel_time(span.get("startTime")); |
| 155 | + let end = parse_otel_time(span.get("endTime")); |
| 156 | + match (start, end) { |
| 157 | + (Some(s), Some(e)) => (e - s).max(0.0), |
| 158 | + _ => 0.0, |
| 159 | + } |
| 160 | +} |
| 161 | + |
| 162 | +/// Parse an OTel `[seconds, nanoseconds]` time array into seconds. |
| 163 | +fn parse_otel_time(value: Option<&Value>) -> Option<f64> { |
| 164 | + let arr = value?.as_array()?; |
| 165 | + let secs = arr.first()?.as_f64()?; |
| 166 | + let nanos = arr.get(1)?.as_f64().unwrap_or(0.0); |
| 167 | + Some(secs + nanos / 1_000_000_000.0) |
| 168 | +} |
| 169 | + |
| 170 | +/// Format seconds as human-readable duration (e.g., "4m 32s"). |
| 171 | +fn format_duration(seconds: f64) -> String { |
| 172 | + let total_secs = seconds.round() as u64; |
| 173 | + if total_secs < 60 { |
| 174 | + format!("{}s", total_secs) |
| 175 | + } else if total_secs < 3600 { |
| 176 | + format!("{}m {}s", total_secs / 60, total_secs % 60) |
| 177 | + } else { |
| 178 | + format!( |
| 179 | + "{}h {}m {}s", |
| 180 | + total_secs / 3600, |
| 181 | + (total_secs % 3600) / 60, |
| 182 | + total_secs % 60 |
| 183 | + ) |
| 184 | + } |
| 185 | +} |
| 186 | + |
| 187 | +/// Format a number with comma separators (e.g., 45230 → "45,230"). |
| 188 | +fn format_number(n: u64) -> String { |
| 189 | + let s = n.to_string(); |
| 190 | + let mut result = String::new(); |
| 191 | + for (i, c) in s.chars().rev().enumerate() { |
| 192 | + if i > 0 && i % 3 == 0 { |
| 193 | + result.push(','); |
| 194 | + } |
| 195 | + result.push(c); |
| 196 | + } |
| 197 | + result.chars().rev().collect() |
| 198 | +} |
| 199 | + |
| 200 | +#[cfg(test)] |
| 201 | +mod tests { |
| 202 | + use super::*; |
| 203 | + |
| 204 | + #[test] |
| 205 | + fn test_format_duration_seconds() { |
| 206 | + assert_eq!(format_duration(0.0), "0s"); |
| 207 | + assert_eq!(format_duration(45.0), "45s"); |
| 208 | + assert_eq!(format_duration(59.4), "59s"); |
| 209 | + } |
| 210 | + |
| 211 | + #[test] |
| 212 | + fn test_format_duration_minutes() { |
| 213 | + assert_eq!(format_duration(60.0), "1m 0s"); |
| 214 | + assert_eq!(format_duration(272.0), "4m 32s"); |
| 215 | + assert_eq!(format_duration(3599.0), "59m 59s"); |
| 216 | + } |
| 217 | + |
| 218 | + #[test] |
| 219 | + fn test_format_duration_hours() { |
| 220 | + assert_eq!(format_duration(3600.0), "1h 0m 0s"); |
| 221 | + assert_eq!(format_duration(7384.0), "2h 3m 4s"); |
| 222 | + } |
| 223 | + |
| 224 | + #[test] |
| 225 | + fn test_format_number() { |
| 226 | + assert_eq!(format_number(0), "0"); |
| 227 | + assert_eq!(format_number(999), "999"); |
| 228 | + assert_eq!(format_number(1000), "1,000"); |
| 229 | + assert_eq!(format_number(45230), "45,230"); |
| 230 | + assert_eq!(format_number(1234567), "1,234,567"); |
| 231 | + } |
| 232 | + |
| 233 | + #[test] |
| 234 | + fn test_from_otel_entries_empty() { |
| 235 | + let stats = AgentStats::from_otel_entries(&[], "test-agent").unwrap(); |
| 236 | + assert_eq!(stats.agent_name, "test-agent"); |
| 237 | + assert_eq!(stats.input_tokens, 0); |
| 238 | + assert_eq!(stats.output_tokens, 0); |
| 239 | + assert_eq!(stats.tool_calls, 0); |
| 240 | + assert_eq!(stats.turns, 0); |
| 241 | + assert!(stats.model.is_none()); |
| 242 | + } |
| 243 | + |
| 244 | + #[test] |
| 245 | + fn test_from_otel_entries_real_fixture() { |
| 246 | + let content = include_str!("../tests/fixtures/copilot-otel.jsonl"); |
| 247 | + let entries = crate::ndjson::parse_ndjson(content).unwrap(); |
| 248 | + let stats = AgentStats::from_otel_entries(&entries, "test-agent").unwrap(); |
| 249 | + |
| 250 | + assert_eq!(stats.model.as_deref(), Some("claude-sonnet-4.5")); |
| 251 | + assert_eq!(stats.input_tokens, 32949); |
| 252 | + assert_eq!(stats.output_tokens, 236); |
| 253 | + assert_eq!(stats.total_tokens(), 33185); |
| 254 | + assert_eq!(stats.turns, 2); |
| 255 | + // execute_tool spans: report_intent + bash = 2 |
| 256 | + assert_eq!(stats.tool_calls, 2); |
| 257 | + // Duration should be ~8 seconds (from the last invoke_agent span) |
| 258 | + assert!(stats.duration_seconds > 7.0 && stats.duration_seconds < 10.0); |
| 259 | + } |
| 260 | + |
| 261 | + #[test] |
| 262 | + fn test_to_markdown_contains_key_elements() { |
| 263 | + let stats = AgentStats { |
| 264 | + agent_name: "Daily Code Review".to_string(), |
| 265 | + model: Some("claude-opus-4.5".to_string()), |
| 266 | + input_tokens: 45230, |
| 267 | + output_tokens: 12450, |
| 268 | + duration_seconds: 272.0, |
| 269 | + tool_calls: 23, |
| 270 | + turns: 8, |
| 271 | + }; |
| 272 | + let md = stats.to_markdown(); |
| 273 | + assert!(md.contains("<details>")); |
| 274 | + assert!(md.contains("Daily Code Review")); |
| 275 | + assert!(md.contains("claude-opus-4.5")); |
| 276 | + assert!(md.contains("45,230")); |
| 277 | + assert!(md.contains("12,450")); |
| 278 | + assert!(md.contains("57,680")); |
| 279 | + assert!(md.contains("4m 32s")); |
| 280 | + assert!(md.contains("23")); |
| 281 | + assert!(md.contains("8")); |
| 282 | + } |
| 283 | + |
| 284 | + #[test] |
| 285 | + fn test_parse_otel_time() { |
| 286 | + // [1776287701, 726000000] = epoch seconds + nanoseconds |
| 287 | + let val = serde_json::json!([1776287701, 726000000]); |
| 288 | + let t = parse_otel_time(Some(&val)).unwrap(); |
| 289 | + assert!((t - 1776287701.726).abs() < 0.001); |
| 290 | + } |
| 291 | + |
| 292 | + #[test] |
| 293 | + fn test_compute_duration_from_span() { |
| 294 | + let span = serde_json::json!({ |
| 295 | + "startTime": [1776287701, 726000000], |
| 296 | + "endTime": [1776287710, 8631000] |
| 297 | + }); |
| 298 | + let d = compute_duration(&span); |
| 299 | + assert!((d - 8.282631).abs() < 0.01); |
| 300 | + } |
| 301 | +} |
0 commit comments