Skip to content

Commit e8154bd

Browse files
jamesadevineCopilot
andcommitted
feat: add OTel JSONL parser for agent statistics
Create src/agent_stats.rs with AgentStats struct that parses Copilot CLI OpenTelemetry file exporter output. Extracts model, token usage, duration, tool calls, and turns from the last invoke_agent span and execute_tool span counts. Reuses ndjson::read_ndjson_file for JSONL parsing. Includes to_markdown() renderer with collapsible details block. 9 unit tests including real copilot-otel.jsonl fixture. Closes: part of #168 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent ed93aff commit e8154bd

3 files changed

Lines changed: 318 additions & 0 deletions

File tree

src/agent_stats.rs

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
//! Agent statistics extracted from Copilot CLI OpenTelemetry output.
2+
//!
3+
//! The Copilot CLI can write OTel spans and metrics to a JSONL file via
4+
//! `COPILOT_OTEL_FILE_EXPORTER_PATH`. This module parses that file to
5+
//! extract agent execution statistics (token usage, duration, model,
6+
//! tool calls, turns) for inclusion in safe output write actions.
7+
8+
use anyhow::{Context, Result};
9+
use serde_json::Value;
10+
use std::path::Path;
11+
12+
/// Agent execution statistics parsed from OTel JSONL.
13+
#[derive(Debug, Clone)]
14+
pub struct AgentStats {
15+
/// Agent name from front matter.
16+
pub agent_name: String,
17+
/// AI model used (e.g., "claude-sonnet-4.5").
18+
pub model: Option<String>,
19+
/// Total input tokens across all LLM calls.
20+
pub input_tokens: u64,
21+
/// Total output tokens across all LLM calls.
22+
pub output_tokens: u64,
23+
/// Wall-clock duration in seconds.
24+
pub duration_seconds: f64,
25+
/// Number of tool invocations.
26+
pub tool_calls: u64,
27+
/// Number of LLM round-trips (turns).
28+
pub turns: u64,
29+
}
30+
31+
/// OTel JSONL filename written by Copilot CLI.
32+
pub const OTEL_FILENAME: &str = "otel.jsonl";
33+
34+
impl AgentStats {
35+
/// Parse agent stats from an OTel JSONL file.
36+
///
37+
/// Uses [`crate::ndjson::read_ndjson_file`] for file I/O, then
38+
/// extracts stats from the parsed entries.
39+
pub async fn from_otel_file(path: &Path, agent_name: &str) -> Result<Self> {
40+
let entries = crate::ndjson::read_ndjson_file(path)
41+
.await
42+
.with_context(|| format!("Failed to read OTel file: {}", path.display()))?;
43+
Self::from_otel_entries(&entries, agent_name)
44+
}
45+
46+
/// Extract stats from pre-parsed OTel JSONL entries.
47+
///
48+
/// Looks for:
49+
/// - The last `invoke_agent` span for aggregated tokens, model, turns, duration
50+
/// - `execute_tool` spans for tool call count
51+
pub fn from_otel_entries(entries: &[Value], agent_name: &str) -> Result<Self> {
52+
let mut stats = AgentStats {
53+
agent_name: agent_name.to_string(),
54+
model: None,
55+
input_tokens: 0,
56+
output_tokens: 0,
57+
duration_seconds: 0.0,
58+
tool_calls: 0,
59+
turns: 0,
60+
};
61+
62+
// Find the last invoke_agent span (contains aggregated totals)
63+
let last_agent_span = entries
64+
.iter()
65+
.filter(|e| {
66+
e.get("type").and_then(|t| t.as_str()) == Some("span")
67+
&& e.get("name").and_then(|n| n.as_str()) == Some("invoke_agent")
68+
})
69+
.last();
70+
71+
if let Some(span) = last_agent_span {
72+
let attrs = span.get("attributes").cloned().unwrap_or(Value::Null);
73+
74+
// Model
75+
stats.model = attrs
76+
.get("gen_ai.request.model")
77+
.and_then(|v| v.as_str())
78+
.map(|s| s.to_string());
79+
80+
// Tokens (aggregated across all chat spans)
81+
stats.input_tokens = attrs
82+
.get("gen_ai.usage.input_tokens")
83+
.and_then(|v| v.as_u64())
84+
.unwrap_or(0);
85+
stats.output_tokens = attrs
86+
.get("gen_ai.usage.output_tokens")
87+
.and_then(|v| v.as_u64())
88+
.unwrap_or(0);
89+
90+
// Turns
91+
stats.turns = attrs
92+
.get("github.copilot.turn_count")
93+
.and_then(|v| v.as_u64())
94+
.unwrap_or(0);
95+
96+
// Duration from startTime/endTime ([seconds, nanoseconds] arrays)
97+
stats.duration_seconds = compute_duration(span);
98+
}
99+
100+
// Count execute_tool spans
101+
stats.tool_calls = entries
102+
.iter()
103+
.filter(|e| {
104+
e.get("type").and_then(|t| t.as_str()) == Some("span")
105+
&& e.get("name")
106+
.and_then(|n| n.as_str())
107+
.is_some_and(|n| n.starts_with("execute_tool"))
108+
})
109+
.count() as u64;
110+
111+
Ok(stats)
112+
}
113+
114+
/// Total tokens (input + output).
115+
pub fn total_tokens(&self) -> u64 {
116+
self.input_tokens + self.output_tokens
117+
}
118+
119+
/// Render as a collapsible markdown stats block.
120+
pub fn to_markdown(&self) -> String {
121+
let duration = format_duration(self.duration_seconds);
122+
let model = self.model.as_deref().unwrap_or("unknown");
123+
124+
format!(
125+
"\n---\n\
126+
<details>\n\
127+
<summary>\u{1F916} Agent Stats ({name})</summary>\n\
128+
\n\
129+
| Metric | Value |\n\
130+
|--------|-------|\n\
131+
| Model | {model} |\n\
132+
| Tokens | {input} input / {output} output ({total} total) |\n\
133+
| Duration | {duration} |\n\
134+
| Tool calls | {tools} |\n\
135+
| Turns | {turns} |\n\
136+
\n\
137+
</details>\n",
138+
name = self.agent_name,
139+
model = model,
140+
input = format_number(self.input_tokens),
141+
output = format_number(self.output_tokens),
142+
total = format_number(self.total_tokens()),
143+
duration = duration,
144+
tools = self.tool_calls,
145+
turns = self.turns,
146+
)
147+
}
148+
}
149+
150+
/// Compute duration from OTel span startTime/endTime.
151+
///
152+
/// Times are `[seconds, nanoseconds]` arrays.
153+
fn compute_duration(span: &Value) -> f64 {
154+
let start = parse_otel_time(span.get("startTime"));
155+
let end = parse_otel_time(span.get("endTime"));
156+
match (start, end) {
157+
(Some(s), Some(e)) => (e - s).max(0.0),
158+
_ => 0.0,
159+
}
160+
}
161+
162+
/// Parse an OTel `[seconds, nanoseconds]` time array into seconds.
163+
fn parse_otel_time(value: Option<&Value>) -> Option<f64> {
164+
let arr = value?.as_array()?;
165+
let secs = arr.first()?.as_f64()?;
166+
let nanos = arr.get(1)?.as_f64().unwrap_or(0.0);
167+
Some(secs + nanos / 1_000_000_000.0)
168+
}
169+
170+
/// Format seconds as human-readable duration (e.g., "4m 32s").
171+
fn format_duration(seconds: f64) -> String {
172+
let total_secs = seconds.round() as u64;
173+
if total_secs < 60 {
174+
format!("{}s", total_secs)
175+
} else if total_secs < 3600 {
176+
format!("{}m {}s", total_secs / 60, total_secs % 60)
177+
} else {
178+
format!(
179+
"{}h {}m {}s",
180+
total_secs / 3600,
181+
(total_secs % 3600) / 60,
182+
total_secs % 60
183+
)
184+
}
185+
}
186+
187+
/// Format a number with comma separators (e.g., 45230 → "45,230").
188+
fn format_number(n: u64) -> String {
189+
let s = n.to_string();
190+
let mut result = String::new();
191+
for (i, c) in s.chars().rev().enumerate() {
192+
if i > 0 && i % 3 == 0 {
193+
result.push(',');
194+
}
195+
result.push(c);
196+
}
197+
result.chars().rev().collect()
198+
}
199+
200+
#[cfg(test)]
201+
mod tests {
202+
use super::*;
203+
204+
#[test]
205+
fn test_format_duration_seconds() {
206+
assert_eq!(format_duration(0.0), "0s");
207+
assert_eq!(format_duration(45.0), "45s");
208+
assert_eq!(format_duration(59.4), "59s");
209+
}
210+
211+
#[test]
212+
fn test_format_duration_minutes() {
213+
assert_eq!(format_duration(60.0), "1m 0s");
214+
assert_eq!(format_duration(272.0), "4m 32s");
215+
assert_eq!(format_duration(3599.0), "59m 59s");
216+
}
217+
218+
#[test]
219+
fn test_format_duration_hours() {
220+
assert_eq!(format_duration(3600.0), "1h 0m 0s");
221+
assert_eq!(format_duration(7384.0), "2h 3m 4s");
222+
}
223+
224+
#[test]
225+
fn test_format_number() {
226+
assert_eq!(format_number(0), "0");
227+
assert_eq!(format_number(999), "999");
228+
assert_eq!(format_number(1000), "1,000");
229+
assert_eq!(format_number(45230), "45,230");
230+
assert_eq!(format_number(1234567), "1,234,567");
231+
}
232+
233+
#[test]
234+
fn test_from_otel_entries_empty() {
235+
let stats = AgentStats::from_otel_entries(&[], "test-agent").unwrap();
236+
assert_eq!(stats.agent_name, "test-agent");
237+
assert_eq!(stats.input_tokens, 0);
238+
assert_eq!(stats.output_tokens, 0);
239+
assert_eq!(stats.tool_calls, 0);
240+
assert_eq!(stats.turns, 0);
241+
assert!(stats.model.is_none());
242+
}
243+
244+
#[test]
245+
fn test_from_otel_entries_real_fixture() {
246+
let content = include_str!("../tests/fixtures/copilot-otel.jsonl");
247+
let entries = crate::ndjson::parse_ndjson(content).unwrap();
248+
let stats = AgentStats::from_otel_entries(&entries, "test-agent").unwrap();
249+
250+
assert_eq!(stats.model.as_deref(), Some("claude-sonnet-4.5"));
251+
assert_eq!(stats.input_tokens, 32949);
252+
assert_eq!(stats.output_tokens, 236);
253+
assert_eq!(stats.total_tokens(), 33185);
254+
assert_eq!(stats.turns, 2);
255+
// execute_tool spans: report_intent + bash = 2
256+
assert_eq!(stats.tool_calls, 2);
257+
// Duration should be ~8 seconds (from the last invoke_agent span)
258+
assert!(stats.duration_seconds > 7.0 && stats.duration_seconds < 10.0);
259+
}
260+
261+
#[test]
262+
fn test_to_markdown_contains_key_elements() {
263+
let stats = AgentStats {
264+
agent_name: "Daily Code Review".to_string(),
265+
model: Some("claude-opus-4.5".to_string()),
266+
input_tokens: 45230,
267+
output_tokens: 12450,
268+
duration_seconds: 272.0,
269+
tool_calls: 23,
270+
turns: 8,
271+
};
272+
let md = stats.to_markdown();
273+
assert!(md.contains("<details>"));
274+
assert!(md.contains("Daily Code Review"));
275+
assert!(md.contains("claude-opus-4.5"));
276+
assert!(md.contains("45,230"));
277+
assert!(md.contains("12,450"));
278+
assert!(md.contains("57,680"));
279+
assert!(md.contains("4m 32s"));
280+
assert!(md.contains("23"));
281+
assert!(md.contains("8"));
282+
}
283+
284+
#[test]
285+
fn test_parse_otel_time() {
286+
// [1776287701, 726000000] = epoch seconds + nanoseconds
287+
let val = serde_json::json!([1776287701, 726000000]);
288+
let t = parse_otel_time(Some(&val)).unwrap();
289+
assert!((t - 1776287701.726).abs() < 0.001);
290+
}
291+
292+
#[test]
293+
fn test_compute_duration_from_span() {
294+
let span = serde_json::json!({
295+
"startTime": [1776287701, 726000000],
296+
"endTime": [1776287710, 8631000]
297+
});
298+
let d = compute_duration(&span);
299+
assert!((d - 8.282631).abs() < 0.01);
300+
}
301+
}

src/main.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
mod agent_stats;
12
mod allowed_hosts;
23
mod compile;
34
mod configure;

0 commit comments

Comments
 (0)