From e8154bd7d678420cbcdfc562bb1957a66b53e11d Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 22:20:24 +0100 Subject: [PATCH 1/8] feat: add OTel JSONL parser for agent statistics Create src/agent_stats.rs with AgentStats struct that parses Copilot CLI OpenTelemetry file exporter output. Extracts model, token usage, duration, tool calls, and turns from the last invoke_agent span and execute_tool span counts. Reuses ndjson::read_ndjson_file for JSONL parsing. Includes to_markdown() renderer with collapsible details block. 9 unit tests including real copilot-otel.jsonl fixture. Closes: part of #168 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agent_stats.rs | 301 ++++++++++++++++++++++++++++++ src/main.rs | 1 + tests/fixtures/copilot-otel.jsonl | 16 ++ 3 files changed, 318 insertions(+) create mode 100644 src/agent_stats.rs create mode 100644 tests/fixtures/copilot-otel.jsonl diff --git a/src/agent_stats.rs b/src/agent_stats.rs new file mode 100644 index 00000000..2f4bafe6 --- /dev/null +++ b/src/agent_stats.rs @@ -0,0 +1,301 @@ +//! Agent statistics extracted from Copilot CLI OpenTelemetry output. +//! +//! The Copilot CLI can write OTel spans and metrics to a JSONL file via +//! `COPILOT_OTEL_FILE_EXPORTER_PATH`. This module parses that file to +//! extract agent execution statistics (token usage, duration, model, +//! tool calls, turns) for inclusion in safe output write actions. + +use anyhow::{Context, Result}; +use serde_json::Value; +use std::path::Path; + +/// Agent execution statistics parsed from OTel JSONL. +#[derive(Debug, Clone)] +pub struct AgentStats { + /// Agent name from front matter. + pub agent_name: String, + /// AI model used (e.g., "claude-sonnet-4.5"). + pub model: Option, + /// Total input tokens across all LLM calls. + pub input_tokens: u64, + /// Total output tokens across all LLM calls. + pub output_tokens: u64, + /// Wall-clock duration in seconds. + pub duration_seconds: f64, + /// Number of tool invocations. + pub tool_calls: u64, + /// Number of LLM round-trips (turns). + pub turns: u64, +} + +/// OTel JSONL filename written by Copilot CLI. +pub const OTEL_FILENAME: &str = "otel.jsonl"; + +impl AgentStats { + /// Parse agent stats from an OTel JSONL file. + /// + /// Uses [`crate::ndjson::read_ndjson_file`] for file I/O, then + /// extracts stats from the parsed entries. + pub async fn from_otel_file(path: &Path, agent_name: &str) -> Result { + let entries = crate::ndjson::read_ndjson_file(path) + .await + .with_context(|| format!("Failed to read OTel file: {}", path.display()))?; + Self::from_otel_entries(&entries, agent_name) + } + + /// Extract stats from pre-parsed OTel JSONL entries. + /// + /// Looks for: + /// - The last `invoke_agent` span for aggregated tokens, model, turns, duration + /// - `execute_tool` spans for tool call count + pub fn from_otel_entries(entries: &[Value], agent_name: &str) -> Result { + let mut stats = AgentStats { + agent_name: agent_name.to_string(), + model: None, + input_tokens: 0, + output_tokens: 0, + duration_seconds: 0.0, + tool_calls: 0, + turns: 0, + }; + + // Find the last invoke_agent span (contains aggregated totals) + let last_agent_span = entries + .iter() + .filter(|e| { + e.get("type").and_then(|t| t.as_str()) == Some("span") + && e.get("name").and_then(|n| n.as_str()) == Some("invoke_agent") + }) + .last(); + + if let Some(span) = last_agent_span { + let attrs = span.get("attributes").cloned().unwrap_or(Value::Null); + + // Model + stats.model = attrs + .get("gen_ai.request.model") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + // Tokens (aggregated across all chat spans) + stats.input_tokens = attrs + .get("gen_ai.usage.input_tokens") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + stats.output_tokens = attrs + .get("gen_ai.usage.output_tokens") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + + // Turns + stats.turns = attrs + .get("github.copilot.turn_count") + .and_then(|v| v.as_u64()) + .unwrap_or(0); + + // Duration from startTime/endTime ([seconds, nanoseconds] arrays) + stats.duration_seconds = compute_duration(span); + } + + // Count execute_tool spans + stats.tool_calls = entries + .iter() + .filter(|e| { + e.get("type").and_then(|t| t.as_str()) == Some("span") + && e.get("name") + .and_then(|n| n.as_str()) + .is_some_and(|n| n.starts_with("execute_tool")) + }) + .count() as u64; + + Ok(stats) + } + + /// Total tokens (input + output). + pub fn total_tokens(&self) -> u64 { + self.input_tokens + self.output_tokens + } + + /// Render as a collapsible markdown stats block. + pub fn to_markdown(&self) -> String { + let duration = format_duration(self.duration_seconds); + let model = self.model.as_deref().unwrap_or("unknown"); + + format!( + "\n---\n\ +
\n\ + \u{1F916} Agent Stats ({name})\n\ + \n\ + | Metric | Value |\n\ + |--------|-------|\n\ + | Model | {model} |\n\ + | Tokens | {input} input / {output} output ({total} total) |\n\ + | Duration | {duration} |\n\ + | Tool calls | {tools} |\n\ + | Turns | {turns} |\n\ + \n\ +
\n", + name = self.agent_name, + model = model, + input = format_number(self.input_tokens), + output = format_number(self.output_tokens), + total = format_number(self.total_tokens()), + duration = duration, + tools = self.tool_calls, + turns = self.turns, + ) + } +} + +/// Compute duration from OTel span startTime/endTime. +/// +/// Times are `[seconds, nanoseconds]` arrays. +fn compute_duration(span: &Value) -> f64 { + let start = parse_otel_time(span.get("startTime")); + let end = parse_otel_time(span.get("endTime")); + match (start, end) { + (Some(s), Some(e)) => (e - s).max(0.0), + _ => 0.0, + } +} + +/// Parse an OTel `[seconds, nanoseconds]` time array into seconds. +fn parse_otel_time(value: Option<&Value>) -> Option { + let arr = value?.as_array()?; + let secs = arr.first()?.as_f64()?; + let nanos = arr.get(1)?.as_f64().unwrap_or(0.0); + Some(secs + nanos / 1_000_000_000.0) +} + +/// Format seconds as human-readable duration (e.g., "4m 32s"). +fn format_duration(seconds: f64) -> String { + let total_secs = seconds.round() as u64; + if total_secs < 60 { + format!("{}s", total_secs) + } else if total_secs < 3600 { + format!("{}m {}s", total_secs / 60, total_secs % 60) + } else { + format!( + "{}h {}m {}s", + total_secs / 3600, + (total_secs % 3600) / 60, + total_secs % 60 + ) + } +} + +/// Format a number with comma separators (e.g., 45230 → "45,230"). +fn format_number(n: u64) -> String { + let s = n.to_string(); + let mut result = String::new(); + for (i, c) in s.chars().rev().enumerate() { + if i > 0 && i % 3 == 0 { + result.push(','); + } + result.push(c); + } + result.chars().rev().collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_format_duration_seconds() { + assert_eq!(format_duration(0.0), "0s"); + assert_eq!(format_duration(45.0), "45s"); + assert_eq!(format_duration(59.4), "59s"); + } + + #[test] + fn test_format_duration_minutes() { + assert_eq!(format_duration(60.0), "1m 0s"); + assert_eq!(format_duration(272.0), "4m 32s"); + assert_eq!(format_duration(3599.0), "59m 59s"); + } + + #[test] + fn test_format_duration_hours() { + assert_eq!(format_duration(3600.0), "1h 0m 0s"); + assert_eq!(format_duration(7384.0), "2h 3m 4s"); + } + + #[test] + fn test_format_number() { + assert_eq!(format_number(0), "0"); + assert_eq!(format_number(999), "999"); + assert_eq!(format_number(1000), "1,000"); + assert_eq!(format_number(45230), "45,230"); + assert_eq!(format_number(1234567), "1,234,567"); + } + + #[test] + fn test_from_otel_entries_empty() { + let stats = AgentStats::from_otel_entries(&[], "test-agent").unwrap(); + assert_eq!(stats.agent_name, "test-agent"); + assert_eq!(stats.input_tokens, 0); + assert_eq!(stats.output_tokens, 0); + assert_eq!(stats.tool_calls, 0); + assert_eq!(stats.turns, 0); + assert!(stats.model.is_none()); + } + + #[test] + fn test_from_otel_entries_real_fixture() { + let content = include_str!("../tests/fixtures/copilot-otel.jsonl"); + let entries = crate::ndjson::parse_ndjson(content).unwrap(); + let stats = AgentStats::from_otel_entries(&entries, "test-agent").unwrap(); + + assert_eq!(stats.model.as_deref(), Some("claude-sonnet-4.5")); + assert_eq!(stats.input_tokens, 32949); + assert_eq!(stats.output_tokens, 236); + assert_eq!(stats.total_tokens(), 33185); + assert_eq!(stats.turns, 2); + // execute_tool spans: report_intent + bash = 2 + assert_eq!(stats.tool_calls, 2); + // Duration should be ~8 seconds (from the last invoke_agent span) + assert!(stats.duration_seconds > 7.0 && stats.duration_seconds < 10.0); + } + + #[test] + fn test_to_markdown_contains_key_elements() { + let stats = AgentStats { + agent_name: "Daily Code Review".to_string(), + model: Some("claude-opus-4.5".to_string()), + input_tokens: 45230, + output_tokens: 12450, + duration_seconds: 272.0, + tool_calls: 23, + turns: 8, + }; + let md = stats.to_markdown(); + assert!(md.contains("
")); + assert!(md.contains("Daily Code Review")); + assert!(md.contains("claude-opus-4.5")); + assert!(md.contains("45,230")); + assert!(md.contains("12,450")); + assert!(md.contains("57,680")); + assert!(md.contains("4m 32s")); + assert!(md.contains("23")); + assert!(md.contains("8")); + } + + #[test] + fn test_parse_otel_time() { + // [1776287701, 726000000] = epoch seconds + nanoseconds + let val = serde_json::json!([1776287701, 726000000]); + let t = parse_otel_time(Some(&val)).unwrap(); + assert!((t - 1776287701.726).abs() < 0.001); + } + + #[test] + fn test_compute_duration_from_span() { + let span = serde_json::json!({ + "startTime": [1776287701, 726000000], + "endTime": [1776287710, 8631000] + }); + let d = compute_duration(&span); + assert!((d - 8.282631).abs() < 0.01); + } +} diff --git a/src/main.rs b/src/main.rs index a03518b7..46db9551 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +mod agent_stats; mod allowed_hosts; mod compile; mod configure; diff --git a/tests/fixtures/copilot-otel.jsonl b/tests/fixtures/copilot-otel.jsonl new file mode 100644 index 00000000..41a78807 --- /dev/null +++ b/tests/fixtures/copilot-otel.jsonl @@ -0,0 +1,16 @@ +{"type":"span","traceId":"088a20d78c1d07c3865e2a0530513674","spanId":"c9c2eeffb602d29f","parentSpanId":"063d303f655b34f7","name":"chat claude-sonnet-4.5","kind":2,"startTime":[1776287660,403000000],"endTime":[1776287669,85879708],"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.conversation.id":"2704d56d-7d43-4a77-aecc-f53503afe2e9","gen_ai.response.finish_reasons":["stop"],"gen_ai.response.model":"claude-sonnet-4.5","gen_ai.response.id":"msg_01HgmSHweE6ojTyqT5gWZhKT","gen_ai.usage.input_tokens":16335,"gen_ai.usage.output_tokens":241,"github.copilot.cost":1,"github.copilot.server_duration":8318,"github.copilot.initiator":"user","github.copilot.turn_id":"0","github.copilot.interaction_id":"00414b60-105c-4696-ab8e-b8cfa57cf160"},"status":{"code":0},"events":[{"name":"github.copilot.session.usage_info","attributes":{"github.copilot.token_limit":168000,"github.copilot.current_tokens":15209,"github.copilot.messages_length":2},"time":[1776287660,750748750],"droppedAttributesCount":0}],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"span","traceId":"088a20d78c1d07c3865e2a0530513674","spanId":"063d303f655b34f7","name":"invoke_agent","kind":2,"startTime":[1776287660,398000000],"endTime":[1776287669,91529792],"attributes":{"gen_ai.operation.name":"invoke_agent","gen_ai.provider.name":"github","gen_ai.agent.id":"2704d56d-7d43-4a77-aecc-f53503afe2e9","gen_ai.conversation.id":"2704d56d-7d43-4a77-aecc-f53503afe2e9","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.agent.version":"1.0.27","gen_ai.response.finish_reasons":["stop"],"gen_ai.usage.input_tokens":16335,"gen_ai.usage.output_tokens":241,"github.copilot.cost":1,"github.copilot.turn_count":1},"status":{"code":0},"events":[{"name":"github.copilot.user.message","attributes":{"github.copilot.user.message.source":"user","github.copilot.user.message.interaction_id":"00414b60-105c-4696-ab8e-b8cfa57cf160"},"time":[1776287660,398170792],"droppedAttributesCount":0}],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"metric","name":"gen_ai.client.operation.duration","description":"GenAI operation duration.","unit":"s","dataPoints":[{"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5"},"startTime":[1776287669,85000000],"endTime":[1776287669,164000000],"value":{"min":8.682295250000001,"max":8.682295250000001,"sum":8.682295250000001,"buckets":{"boundaries":[0.01,0.02,0.04,0.08,0.16,0.32,0.64,1.28,2.56,5.12,10.24,20.48,40.96,81.92],"counts":[0,0,0,0,0,0,0,0,0,0,1,0,0,0,0]},"count":1}},{"attributes":{"gen_ai.operation.name":"invoke_agent","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5"},"startTime":[1776287669,91000000],"endTime":[1776287669,164000000],"value":{"min":8.693246542,"max":8.693246542,"sum":8.693246542,"buckets":{"boundaries":[0.01,0.02,0.04,0.08,0.16,0.32,0.64,1.28,2.56,5.12,10.24,20.48,40.96,81.92],"counts":[0,0,0,0,0,0,0,0,0,0,1,0,0,0,0]},"count":1}}]} +{"type":"metric","name":"gen_ai.client.token.usage","description":"Number of input and output tokens used.","unit":"{token}","dataPoints":[{"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5","gen_ai.token.type":"input"},"startTime":[1776287669,85000000],"endTime":[1776287669,164000000],"value":{"min":16335,"max":16335,"sum":16335,"buckets":{"boundaries":[1,4,16,64,256,1024,4096,16384,65536,262144,1048576,4194304,16777216,67108864],"counts":[0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]},"count":1}},{"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5","gen_ai.token.type":"output"},"startTime":[1776287669,85000000],"endTime":[1776287669,164000000],"value":{"min":241,"max":241,"sum":241,"buckets":{"boundaries":[1,4,16,64,256,1024,4096,16384,65536,262144,1048576,4194304,16777216,67108864],"counts":[0,0,0,0,1,0,0,0,0,0,0,0,0,0,0]},"count":1}}]} +{"type":"metric","name":"github.copilot.agent.turn.count","description":"Number of LLM round-trips per agent invocation.","unit":"{turn}","dataPoints":[{"attributes":{"gen_ai.operation.name":"invoke_agent"},"startTime":[1776287669,92000000],"endTime":[1776287669,164000000],"value":{"min":1,"max":1,"sum":1,"buckets":{"boundaries":[0,5,10,25,50,75,100,250,500,750,1000,2500,5000,7500,10000],"counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"count":1}}]} +{"type":"span","traceId":"3184a26c641a6b29bf1f03d5de674e0a","spanId":"f1b63480c6f78981","parentSpanId":"c6b41964c101ae32","name":"execute_tool report_intent","kind":0,"startTime":[1776287707,467000000],"endTime":[1776287707,479244083],"attributes":{"gen_ai.operation.name":"execute_tool","gen_ai.tool.name":"report_intent","gen_ai.tool.call.id":"toolu_01Dz6cwy1p1jixbLe8tUgLMP","gen_ai.tool.type":"function","gen_ai.provider.name":"github"},"status":{"code":0},"events":[],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"span","traceId":"3184a26c641a6b29bf1f03d5de674e0a","spanId":"9ee8976761e23b73","parentSpanId":"6bf31ab946b8a3d7","name":"permission","kind":0,"startTime":[1776287707,514000000],"endTime":[1776287707,514134792],"attributes":{"github.copilot.permission.kind":"shell","github.copilot.permission.result":"approved"},"status":{"code":0},"events":[],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"span","traceId":"3184a26c641a6b29bf1f03d5de674e0a","spanId":"6bf31ab946b8a3d7","parentSpanId":"c6b41964c101ae32","name":"execute_tool bash","kind":0,"startTime":[1776287707,467000000],"endTime":[1776287707,935016625],"attributes":{"gen_ai.operation.name":"execute_tool","gen_ai.tool.name":"bash","gen_ai.tool.call.id":"toolu_01FQwme51sxxwfEEHhvYMCHV","gen_ai.tool.type":"function","gen_ai.provider.name":"github"},"status":{"code":0},"events":[],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"span","traceId":"3184a26c641a6b29bf1f03d5de674e0a","spanId":"292baa9046013b62","parentSpanId":"c6b41964c101ae32","name":"chat claude-sonnet-4.5","kind":2,"startTime":[1776287701,730000000],"endTime":[1776287707,972193125],"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.conversation.id":"f02e2d21-f332-484f-96f3-b25e345372e2","gen_ai.response.finish_reasons":["stop"],"gen_ai.response.model":"claude-sonnet-4.5","gen_ai.response.id":"msg_017RrqSn4b2pvdYbbkuPvEby","gen_ai.usage.input_tokens":16331,"gen_ai.usage.output_tokens":216,"gen_ai.usage.cache_read.input_tokens":6966,"github.copilot.cost":1,"github.copilot.server_duration":5397,"github.copilot.initiator":"user","github.copilot.turn_id":"0","github.copilot.interaction_id":"37dff89d-552c-4b8e-bc95-8d7941b93dff"},"status":{"code":0},"events":[{"name":"github.copilot.session.usage_info","attributes":{"github.copilot.token_limit":168000,"github.copilot.current_tokens":15208,"github.copilot.messages_length":2},"time":[1776287702,60098541],"droppedAttributesCount":0},{"name":"github.copilot.hook.start","attributes":{"github.copilot.hook.type":"postToolUse","github.copilot.hook.invocation_id":"5c9e9898-6ee7-4800-a3b2-bd6cc9acebc2"},"time":[1776287707,476934541],"droppedAttributesCount":0},{"name":"github.copilot.hook.end","attributes":{"github.copilot.hook.type":"postToolUse","github.copilot.hook.invocation_id":"5c9e9898-6ee7-4800-a3b2-bd6cc9acebc2"},"time":[1776287707,477132666],"droppedAttributesCount":0},{"name":"github.copilot.hook.start","attributes":{"github.copilot.hook.type":"postToolUse","github.copilot.hook.invocation_id":"3117cb04-3b45-455e-a4bf-a5558c6f6c22"},"time":[1776287707,931591625],"droppedAttributesCount":0},{"name":"github.copilot.hook.end","attributes":{"github.copilot.hook.type":"postToolUse","github.copilot.hook.invocation_id":"3117cb04-3b45-455e-a4bf-a5558c6f6c22"},"time":[1776287707,932060000],"droppedAttributesCount":0}],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"span","traceId":"3184a26c641a6b29bf1f03d5de674e0a","spanId":"122dedda8e545dfb","parentSpanId":"c6b41964c101ae32","name":"chat claude-sonnet-4.5","kind":2,"startTime":[1776287707,973000000],"endTime":[1776287710,3925667],"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.conversation.id":"f02e2d21-f332-484f-96f3-b25e345372e2","gen_ai.response.finish_reasons":["stop"],"gen_ai.response.model":"claude-sonnet-4.5","gen_ai.response.id":"msg_015TKLGnQbj3wLPMse5ktzaM","gen_ai.usage.input_tokens":16618,"gen_ai.usage.output_tokens":20,"gen_ai.usage.cache_read.input_tokens":16321,"github.copilot.cost":1,"github.copilot.server_duration":1984,"github.copilot.initiator":"agent","github.copilot.turn_id":"1","github.copilot.interaction_id":"37dff89d-552c-4b8e-bc95-8d7941b93dff"},"status":{"code":0},"events":[{"name":"github.copilot.session.usage_info","attributes":{"github.copilot.token_limit":168000,"github.copilot.current_tokens":15492,"github.copilot.messages_length":5},"time":[1776287708,13043958],"droppedAttributesCount":0}],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"span","traceId":"3184a26c641a6b29bf1f03d5de674e0a","spanId":"c6b41964c101ae32","name":"invoke_agent","kind":2,"startTime":[1776287701,726000000],"endTime":[1776287710,8631000],"attributes":{"gen_ai.operation.name":"invoke_agent","gen_ai.provider.name":"github","gen_ai.agent.id":"f02e2d21-f332-484f-96f3-b25e345372e2","gen_ai.conversation.id":"f02e2d21-f332-484f-96f3-b25e345372e2","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.agent.version":"1.0.27","gen_ai.response.finish_reasons":["stop"],"gen_ai.usage.input_tokens":32949,"gen_ai.usage.output_tokens":236,"gen_ai.usage.cache_read.input_tokens":23287,"github.copilot.cost":2,"github.copilot.turn_count":2},"status":{"code":0},"events":[{"name":"github.copilot.user.message","attributes":{"github.copilot.user.message.source":"user","github.copilot.user.message.interaction_id":"37dff89d-552c-4b8e-bc95-8d7941b93dff"},"time":[1776287701,726176750],"droppedAttributesCount":0}],"resource":{"attributes":{"service.name":"github-copilot","service.version":"1.0.27"},"schemaUrl":"https://opentelemetry.io/schemas/1.40.0"},"instrumentationScope":{"name":"github.copilot","version":"1.0.27"}} +{"type":"metric","name":"gen_ai.client.operation.duration","description":"GenAI operation duration.","unit":"s","dataPoints":[{"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5"},"startTime":[1776287707,972000000],"endTime":[1776287710,65000000],"value":{"min":2.030883290999999,"max":6.2421425,"sum":8.273025790999998,"buckets":{"boundaries":[0.01,0.02,0.04,0.08,0.16,0.32,0.64,1.28,2.56,5.12,10.24,20.48,40.96,81.92],"counts":[0,0,0,0,0,0,0,0,1,0,1,0,0,0,0]},"count":2}},{"attributes":{"gen_ai.operation.name":"invoke_agent","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5"},"startTime":[1776287710,8000000],"endTime":[1776287710,65000000],"value":{"min":8.282399042000002,"max":8.282399042000002,"sum":8.282399042000002,"buckets":{"boundaries":[0.01,0.02,0.04,0.08,0.16,0.32,0.64,1.28,2.56,5.12,10.24,20.48,40.96,81.92],"counts":[0,0,0,0,0,0,0,0,0,0,1,0,0,0,0]},"count":1}}]} +{"type":"metric","name":"gen_ai.client.token.usage","description":"Number of input and output tokens used.","unit":"{token}","dataPoints":[{"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5","gen_ai.token.type":"input"},"startTime":[1776287707,972000000],"endTime":[1776287710,65000000],"value":{"min":16331,"max":16618,"sum":32949,"buckets":{"boundaries":[1,4,16,64,256,1024,4096,16384,65536,262144,1048576,4194304,16777216,67108864],"counts":[0,0,0,0,0,0,0,1,1,0,0,0,0,0,0]},"count":2}},{"attributes":{"gen_ai.operation.name":"chat","gen_ai.provider.name":"github","gen_ai.request.model":"claude-sonnet-4.5","gen_ai.response.model":"claude-sonnet-4.5","gen_ai.token.type":"output"},"startTime":[1776287707,972000000],"endTime":[1776287710,65000000],"value":{"min":20,"max":216,"sum":236,"buckets":{"boundaries":[1,4,16,64,256,1024,4096,16384,65536,262144,1048576,4194304,16777216,67108864],"counts":[0,0,0,1,1,0,0,0,0,0,0,0,0,0,0]},"count":2}}]} +{"type":"metric","name":"github.copilot.tool.call.count","description":"Number of tool invocations by tool name and outcome.","unit":"{call}","dataPoints":[{"attributes":{"gen_ai.tool.name":"report_intent","success":true},"startTime":[1776287707,481000000],"endTime":[1776287710,65000000],"value":1},{"attributes":{"gen_ai.tool.name":"bash","success":true},"startTime":[1776287707,936000000],"endTime":[1776287710,65000000],"value":1}]} +{"type":"metric","name":"github.copilot.tool.call.duration","description":"Tool execution latency.","unit":"s","dataPoints":[{"attributes":{"gen_ai.tool.name":"report_intent"},"startTime":[1776287707,481000000],"endTime":[1776287710,65000000],"value":{"min":0.013458832999999686,"max":0.013458832999999686,"sum":0.013458832999999686,"buckets":{"boundaries":[0.01,0.02,0.04,0.08,0.16,0.32,0.64,1.28,2.56,5.12,10.24,20.48,40.96,81.92],"counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,0]},"count":1}},{"attributes":{"gen_ai.tool.name":"bash"},"startTime":[1776287707,936000000],"endTime":[1776287710,65000000],"value":{"min":0.46841050000000084,"max":0.46841050000000084,"sum":0.46841050000000084,"buckets":{"boundaries":[0.01,0.02,0.04,0.08,0.16,0.32,0.64,1.28,2.56,5.12,10.24,20.48,40.96,81.92],"counts":[0,0,0,0,0,0,1,0,0,0,0,0,0,0,0]},"count":1}}]} +{"type":"metric","name":"github.copilot.agent.turn.count","description":"Number of LLM round-trips per agent invocation.","unit":"{turn}","dataPoints":[{"attributes":{"gen_ai.operation.name":"invoke_agent"},"startTime":[1776287710,8000000],"endTime":[1776287710,65000000],"value":{"min":2,"max":2,"sum":2,"buckets":{"boundaries":[0,5,10,25,50,75,100,250,500,750,1000,2500,5000,7500,10000],"counts":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},"count":1}}]} From 41e5248c6f0b24ddeaa03e54a8b396be6456fab7 Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 22:23:48 +0100 Subject: [PATCH 2/8] feat: plumb agent_name and agent_stats through ExecutionContext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add agent_name: Option and agent_stats: Option to ExecutionContext - Plumb front_matter.name to ctx.agent_name in main.rs execute handler - Load otel.jsonl from safe_output_dir, parse into ctx.agent_stats (non-fatal if missing — just logs debug/warn) - Add OTel env vars to base.yml AWF step (always-on: COPILOT_OTEL_ENABLED, COPILOT_OTEL_FILE_EXPORTER_PATH, COPILOT_OTEL_EXPORTER_TYPE) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/execute.rs | 16 ++++++++++++++++ src/main.rs | 23 +++++++++++++++++++++++ src/safeoutputs/create_wiki_page.rs | 8 ++++++++ src/safeoutputs/result.rs | 6 ++++++ src/safeoutputs/update_wiki_page.rs | 8 ++++++++ src/safeoutputs/update_work_item.rs | 8 ++++++++ templates/base.yml | 3 +++ 7 files changed, 72 insertions(+) diff --git a/src/execute.rs b/src/execute.rs index 295aab7e..303bb53e 100644 --- a/src/execute.rs +++ b/src/execute.rs @@ -514,6 +514,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let result = execute_safe_output(&entry, &ctx).await; @@ -546,6 +548,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let result = execute_safe_output(&entry, &ctx).await; @@ -694,6 +698,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let result = execute_safe_output(&entry, &ctx).await; @@ -736,6 +742,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let result = execute_safe_output(&entry, &ctx).await; @@ -778,6 +786,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let result = execute_safe_output(&entry, &ctx).await; @@ -825,6 +835,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let results = execute_safe_outputs(temp_dir.path(), &ctx).await; @@ -1031,6 +1043,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let results = execute_safe_outputs(temp_dir.path(), &ctx).await; @@ -1074,6 +1088,8 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let results = execute_safe_outputs(temp_dir.path(), &ctx).await.unwrap(); diff --git a/src/main.rs b/src/main.rs index 46db9551..12df306f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -216,6 +216,29 @@ async fn main() -> Result<()> { ctx.working_directory = safe_output_dir.clone(); ctx.tool_configs = front_matter.safe_outputs.clone(); ctx.allowed_repositories = allowed_repositories; + ctx.agent_name = Some(front_matter.name.clone()); + + // Load agent stats from OTel JSONL if available + let otel_path = safe_output_dir.join(agent_stats::OTEL_FILENAME); + if otel_path.exists() { + match agent_stats::AgentStats::from_otel_file(&otel_path, &front_matter.name) + .await + { + Ok(stats) => { + log::info!( + "Agent stats: {} input / {} output tokens, {}s duration, {} tool calls, {} turns", + stats.input_tokens, stats.output_tokens, + stats.duration_seconds as u64, stats.tool_calls, stats.turns + ); + ctx.agent_stats = Some(stats); + } + Err(e) => { + log::warn!("Failed to parse OTel stats file: {}", e); + } + } + } else { + log::debug!("No OTel stats file found at {}", otel_path.display()); + } let results = execute::execute_safe_outputs(&safe_output_dir, &ctx).await?; diff --git a/src/safeoutputs/create_wiki_page.rs b/src/safeoutputs/create_wiki_page.rs index 6de38513..f04b3307 100644 --- a/src/safeoutputs/create_wiki_page.rs +++ b/src/safeoutputs/create_wiki_page.rs @@ -668,6 +668,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: std::collections::HashMap::new(), + agent_name: None, + agent_stats: None, }; // wiki-name not in config → should return Err @@ -731,6 +733,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let outcome = result.execute_impl(&ctx).await.unwrap(); @@ -769,6 +773,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let outcome = result.execute_impl(&ctx).await.unwrap(); @@ -807,6 +813,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; // The GET will fail (network unreachable with a fake host), so the diff --git a/src/safeoutputs/result.rs b/src/safeoutputs/result.rs index a1d11753..d65e2a49 100644 --- a/src/safeoutputs/result.rs +++ b/src/safeoutputs/result.rs @@ -56,6 +56,10 @@ pub struct ExecutionContext { /// Allowed repositories for PRs: "self" + checkout list aliases /// Maps alias to ADO repo name (e.g., "other-repo" -> "org/other-repo") pub allowed_repositories: HashMap, + /// Agent name from front matter (for display in stats blocks) + pub agent_name: Option, + /// Agent execution statistics parsed from OTel JSONL + pub agent_stats: Option, } impl ExecutionContext { @@ -109,6 +113,8 @@ impl Default for ExecutionContext { repository_id: std::env::var("BUILD_REPOSITORY_ID").ok(), repository_name: std::env::var("BUILD_REPOSITORY_NAME").ok(), allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, } } } diff --git a/src/safeoutputs/update_wiki_page.rs b/src/safeoutputs/update_wiki_page.rs index ae44093c..66b3d0fc 100644 --- a/src/safeoutputs/update_wiki_page.rs +++ b/src/safeoutputs/update_wiki_page.rs @@ -638,6 +638,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: std::collections::HashMap::new(), + agent_name: None, + agent_stats: None, }; // wiki-name not in config → should return Err @@ -701,6 +703,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let outcome = result.execute_impl(&ctx).await.unwrap(); @@ -739,6 +743,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let outcome = result.execute_impl(&ctx).await.unwrap(); @@ -777,6 +783,8 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; // The GET will fail (network unreachable with a fake host), so the diff --git a/src/safeoutputs/update_work_item.rs b/src/safeoutputs/update_work_item.rs index de0d939b..94038e1c 100644 --- a/src/safeoutputs/update_work_item.rs +++ b/src/safeoutputs/update_work_item.rs @@ -716,6 +716,8 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let exec_result = result.execute_sanitized(&ctx).await; @@ -765,6 +767,8 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let exec_result = result.execute_sanitized(&ctx).await.unwrap(); @@ -810,6 +814,8 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let exec_result = result.execute_sanitized(&ctx).await.unwrap(); @@ -857,6 +863,8 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), + agent_name: None, + agent_stats: None, }; let exec_result = result.execute_sanitized(&ctx).await.unwrap(); diff --git a/templates/base.yml b/templates/base.yml index f57576ab..1aefbdb0 100644 --- a/templates/base.yml +++ b/templates/base.yml @@ -336,6 +336,9 @@ jobs: {{ copilot_ado_env }} GITHUB_TOKEN: $(GITHUB_TOKEN) GITHUB_READ_ONLY: 1 + COPILOT_OTEL_ENABLED: "true" + COPILOT_OTEL_EXPORTER_TYPE: "file" + COPILOT_OTEL_FILE_EXPORTER_PATH: "/tmp/awf-tools/staging/otel.jsonl" - bash: | # Copy safe outputs from /tmp back to staging for artifact publish From fbfb385e6bc74e74c9e2f85ced87e2ea17516e67 Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 22:28:32 +0100 Subject: [PATCH 3/8] feat: append agent stats to 6 safe output write actions Append a collapsible markdown stats block to safe outputs that produce human-readable content. Each tool reads include_stats from its typed config struct (deserialized via ctx.get_tool_config), matching the existing config pattern used for all other tool options. Safe outputs with stats: - create-pull-request (PR description) - create-work-item (work item description) - comment-on-work-item (comment body) - add-pr-comment (PR comment body) - create-wiki-page (wiki page content) - update-wiki-page (wiki page content) Per-tool opt-out via front matter: safe-outputs: create-pull-request: include-stats: false Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agent_stats.rs | 20 +++++++++++++++++++- src/safeoutputs/add_pr_comment.rs | 15 +++++++++++++++ src/safeoutputs/comment_on_work_item.rs | 15 ++++++++++++++- src/safeoutputs/create_pr.rs | 14 ++++++++++++-- src/safeoutputs/create_wiki_page.rs | 18 +++++++++++++++--- src/safeoutputs/create_work_item.rs | 16 +++++++++++++++- src/safeoutputs/update_wiki_page.rs | 16 +++++++++++++++- 7 files changed, 105 insertions(+), 9 deletions(-) diff --git a/src/agent_stats.rs b/src/agent_stats.rs index 2f4bafe6..56cfbde6 100644 --- a/src/agent_stats.rs +++ b/src/agent_stats.rs @@ -147,7 +147,25 @@ impl AgentStats { } } -/// Compute duration from OTel span startTime/endTime. +/// Append agent stats markdown to a body string if stats are available +/// and stats are not opted out. +/// +/// Used by safe output executors after they read their typed config +/// (which contains the `include_stats` field). +pub fn append_stats_to_body( + body: &str, + ctx: &crate::safeoutputs::ExecutionContext, + include_stats: bool, +) -> String { + if !include_stats { + return body.to_string(); + } + + match &ctx.agent_stats { + Some(stats) => format!("{}{}", body, stats.to_markdown()), + None => body.to_string(), + } +} /// /// Times are `[seconds, nanoseconds]` arrays. fn compute_duration(span: &Value) -> f64 { diff --git a/src/safeoutputs/add_pr_comment.rs b/src/safeoutputs/add_pr_comment.rs index 52e93e2c..46577611 100644 --- a/src/safeoutputs/add_pr_comment.rs +++ b/src/safeoutputs/add_pr_comment.rs @@ -150,6 +150,9 @@ pub struct AddPrCommentConfig { /// If empty, all valid statuses are allowed. #[serde(default, rename = "allowed-statuses")] pub allowed_statuses: Vec, + /// Whether to include agent execution stats in the output (default: true). + #[serde(default = "default_include_stats", rename = "include-stats")] + pub include_stats: bool, } impl Default for AddPrCommentConfig { @@ -158,6 +161,7 @@ impl Default for AddPrCommentConfig { comment_prefix: None, allowed_repositories: Vec::new(), allowed_statuses: Vec::new(), + include_stats: true, } } } @@ -286,6 +290,11 @@ impl Executor for AddPrCommentResult { Some(prefix) => format!("{}{}", prefix, self.content), None => self.content.clone(), }; + let comment_body = crate::agent_stats::append_stats_to_body( + &comment_body, + ctx, + config.include_stats, + ); // Build the API URL let url = format!( @@ -578,6 +587,7 @@ allowed-statuses: comment_prefix: None, allowed_repositories: Vec::new(), allowed_statuses: vec!["Active".to_string(), "Closed".to_string()], + include_stats: true, }; // Test the exact comparison logic extracted from execute_impl let status = "active"; @@ -598,6 +608,7 @@ allowed-statuses: comment_prefix: None, allowed_repositories: Vec::new(), allowed_statuses: vec!["active".to_string()], + include_stats: true, }; let status = "Active"; let matched = config @@ -610,3 +621,7 @@ allowed-statuses: ); } } + +fn default_include_stats() -> bool { + true +} \ No newline at end of file diff --git a/src/safeoutputs/comment_on_work_item.rs b/src/safeoutputs/comment_on_work_item.rs index f53cf0f5..a473acc4 100644 --- a/src/safeoutputs/comment_on_work_item.rs +++ b/src/safeoutputs/comment_on_work_item.rs @@ -100,6 +100,10 @@ pub struct CommentOnWorkItemConfig { /// Target scope — which work items can be commented on. /// `None` means no target was configured; execution must reject this. pub target: Option, + + /// Whether to include agent execution stats in the comment (default: true). + #[serde(default = "default_include_stats", rename = "include-stats")] + pub include_stats: bool, } /// Fetch a work item's area path from the ADO API @@ -258,8 +262,13 @@ impl Executor for CommentOnWorkItemResult { ); debug!("API URL: {}", url); + let body_with_stats = crate::agent_stats::append_stats_to_body( + &self.body, + ctx, + config.include_stats, + ); let comment_body = serde_json::json!({ - "text": self.body, + "text": body_with_stats, }); info!("Sending comment to work item #{}", self.work_item_id); @@ -472,3 +481,7 @@ target: "*" assert!(config.target.is_some()); } } + +fn default_include_stats() -> bool { + true +} diff --git a/src/safeoutputs/create_pr.rs b/src/safeoutputs/create_pr.rs index 7729548f..02a6db77 100644 --- a/src/safeoutputs/create_pr.rs +++ b/src/safeoutputs/create_pr.rs @@ -456,6 +456,10 @@ pub struct CreatePrConfig { /// so operators can manually create the PR. No work item is created automatically. #[serde(default = "default_true", rename = "fallback-record-branch")] pub fallback_record_branch: bool, + + /// Whether to include agent execution stats in the PR description (default: true). + #[serde(default = "default_true", rename = "include-stats")] + pub include_stats: bool, } fn default_target_branch() -> String { @@ -500,6 +504,7 @@ impl Default for CreatePrConfig { labels: Vec::new(), work_items: Vec::new(), fallback_record_branch: true, + include_stats: true, } } } @@ -1255,8 +1260,13 @@ impl Executor for CreatePrResult { } debug!("Changes pushed successfully"); - // Append provenance footer to description + // Append provenance footer and agent stats to description let description_with_footer = format!("{}{}", self.description, generate_pr_footer()); + let description_with_stats = crate::agent_stats::append_stats_to_body( + &description_with_footer, + ctx, + config.include_stats, + ); // Create the pull request via REST API info!("Creating pull request"); @@ -1270,7 +1280,7 @@ impl Executor for CreatePrResult { "sourceRefName": source_ref, "targetRefName": target_ref, "title": effective_title, - "description": description_with_footer, + "description": description_with_stats, "isDraft": config.draft, }); diff --git a/src/safeoutputs/create_wiki_page.rs b/src/safeoutputs/create_wiki_page.rs index f04b3307..354a2546 100644 --- a/src/safeoutputs/create_wiki_page.rs +++ b/src/safeoutputs/create_wiki_page.rs @@ -132,9 +132,11 @@ pub struct CreateWikiPageConfig { /// Default commit comment used when the agent does not supply one. #[serde(default)] pub comment: Option, -} -// ============================================================================ + /// Whether to include agent execution stats in the output (default: true). + #[serde(default = "default_include_stats", rename = "include-stats")] + pub include_stats: bool, +} // Path helpers // ============================================================================ @@ -321,7 +323,13 @@ impl Executor for CreateWikiPageResult { .header("Content-Type", "application/json") .header("If-Match", "") .basic_auth("", Some(token)) - .json(&serde_json::json!({ "content": self.content })) + .json(&serde_json::json!({ + "content": crate::agent_stats::append_stats_to_body( + &self.content, + ctx, + config.include_stats, + ) + })) .send() .await .context("Failed to create wiki page")?; @@ -896,3 +904,7 @@ wiki-name: "MyProject.wiki" assert_eq!(encoded, "MyProject.wiki"); } } + +fn default_include_stats() -> bool { + true +} \ No newline at end of file diff --git a/src/safeoutputs/create_work_item.rs b/src/safeoutputs/create_work_item.rs index 02c8e1b8..d8a4f4db 100644 --- a/src/safeoutputs/create_work_item.rs +++ b/src/safeoutputs/create_work_item.rs @@ -98,6 +98,10 @@ pub struct CreateWorkItemConfig { #[serde(default, rename = "artifact-link")] #[sanitize_config(nested)] pub artifact_link: ArtifactLinkConfig, + + /// Whether to include agent execution stats in the output (default: true). + #[serde(default = "default_include_stats", rename = "include-stats")] + pub include_stats: bool, } /// Configuration for artifact links (repository linking for GitHub Copilot) @@ -141,6 +145,7 @@ impl Default for CreateWorkItemConfig { tags: Vec::new(), custom_fields: std::collections::HashMap::new(), artifact_link: ArtifactLinkConfig::default(), + include_stats: true, } } } @@ -269,9 +274,14 @@ impl Executor for CreateWorkItemResult { debug!("API URL: {}", url); // Build the patch document for work item creation + let description_with_stats = crate::agent_stats::append_stats_to_body( + &self.description, + ctx, + config.include_stats, + ); let mut patch_doc = vec![ field_op("System.Title", &self.title), - field_op("System.Description", &self.description), + field_op("System.Description", &description_with_stats), // Tell Azure DevOps the description is markdown serde_json::json!({ "op": "add", @@ -524,3 +534,7 @@ tags: assert_eq!(config.tags, vec!["my-tag"]); } } + +fn default_include_stats() -> bool { + true +} \ No newline at end of file diff --git a/src/safeoutputs/update_wiki_page.rs b/src/safeoutputs/update_wiki_page.rs index 66b3d0fc..00d739ca 100644 --- a/src/safeoutputs/update_wiki_page.rs +++ b/src/safeoutputs/update_wiki_page.rs @@ -128,6 +128,10 @@ pub struct UpdateWikiPageConfig { /// Default commit comment used when the agent does not supply one. #[serde(default)] pub comment: Option, + + /// Whether to include agent execution stats in the output (default: true). + #[serde(default = "default_include_stats", rename = "include-stats")] + pub include_stats: bool, } // ============================================================================ @@ -316,7 +320,13 @@ impl Executor for UpdateWikiPageResult { .query(&put_query) .header("Content-Type", "application/json") .basic_auth("", Some(token)) - .json(&serde_json::json!({ "content": self.content })); + .json(&serde_json::json!({ + "content": crate::agent_stats::append_stats_to_body( + &self.content, + ctx, + config.include_stats, + ) + })); // Provide the ETag for optimistic concurrency when updating an existing page. if let Some(etag) = &etag { @@ -857,3 +867,7 @@ wiki-name: "MyProject.wiki" assert_eq!(encoded, "MyProject.wiki"); } } + +fn default_include_stats() -> bool { + true +} \ No newline at end of file From 021dad0538dafb5ab18455c594c3b01d33fee7c1 Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 22:49:46 +0100 Subject: [PATCH 4/8] fix: sanitize OTel values, filter internal tools, remove redundant agent_name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Security: - Sanitize model and agent_name before embedding in markdown stats block. Strips control chars, neutralizes ##vso[ pipeline commands, escapes pipe chars that break markdown tables. The OTel file is writable by the agent inside AWF, so these values are untrusted. Bug fix: - Filter out Copilot CLI internal tool spans (report_intent, permission) from tool_calls count. Only user-visible tool invocations are counted. Cleanup: - Remove redundant agent_name from ExecutionContext — AgentStats already stores it. Single source of truth. Docs: - Document include-stats option for all 6 safe outputs in AGENTS.md Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 6 +++ src/agent_stats.rs | 77 ++++++++++++++++++++++++++--- src/execute.rs | 8 --- src/main.rs | 1 - src/safeoutputs/create_wiki_page.rs | 4 -- src/safeoutputs/result.rs | 3 -- src/safeoutputs/update_wiki_page.rs | 4 -- src/safeoutputs/update_work_item.rs | 4 -- 8 files changed, 76 insertions(+), 31 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index ea3f1b9a..752981aa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1022,6 +1022,7 @@ Adds a comment to an existing Azure DevOps work item. This is the ADO equivalent **Configuration options (front matter):** - `max` - Maximum number of comments per run (default: 1) +- `include-stats` - Whether to append agent execution stats to the comment body (default: true) - `target` - **Required** — scoping policy for which work items can be commented on: - `"*"` - Any work item in the project (unrestricted, must be explicit) - `12345` - A specific work item ID @@ -1053,6 +1054,7 @@ Creates an Azure DevOps work item. - `tags` - List of tags to apply - `custom-fields` - Map of custom field reference names to values (e.g., `Custom.MyField: "value"`) - `max` - Maximum number of create-work-item outputs allowed per run (default: 1) +- `include-stats` - Whether to append agent execution stats to the work item description (default: true) - `artifact-link` - Configuration for GitHub Copilot artifact linking: - `enabled` - Whether to add an artifact link (default: false) - `repository` - Repository name override (defaults to BUILD_REPOSITORY_NAME) @@ -1151,6 +1153,7 @@ Note: The source branch name is auto-generated from a sanitized version of the P - `labels` - List of labels to apply - `work-items` - List of work item IDs to link - `max` - Maximum number of create-pull-request outputs allowed per run (default: 1) +- `include-stats` - Whether to append agent execution stats (token usage, duration, model) to the PR description (default: true) **Multi-repository support:** When `workspace: root` and multiple repositories are checked out, agents can create PRs for any allowed repository: @@ -1204,6 +1207,7 @@ safe-outputs: comment-prefix: "[Agent Review] " # Optional — prepended to all comments allowed-repositories: [] # Optional — restrict which repos can be commented on max: 1 # Maximum per run (default: 1) + include-stats: true # Append agent stats to comment (default: true) ``` #### reply-to-pr-comment @@ -1417,6 +1421,7 @@ safe-outputs: title-prefix: "[Agent] " # Optional — prepended to the last path segment (the page title) comment: "Created by agent" # Optional — default commit comment when agent omits one max: 1 # Maximum number of create-wiki-page outputs allowed per run (default: 1) + include-stats: true # Append agent stats to wiki page content (default: true) ``` Note: `wiki-name` is required. If it is not set, execution fails with an explicit error message. @@ -1442,6 +1447,7 @@ safe-outputs: title-prefix: "[Agent] " # Optional — prepended to the last path segment (the page title) comment: "Updated by agent" # Optional — default commit comment when agent omits one max: 1 # Maximum number of update-wiki-page outputs allowed per run (default: 1) + include-stats: true # Append agent stats to wiki page content (default: true) ``` Note: `wiki-name` is required. If it is not set, execution fails with an explicit error message. diff --git a/src/agent_stats.rs b/src/agent_stats.rs index 56cfbde6..0b55bb26 100644 --- a/src/agent_stats.rs +++ b/src/agent_stats.rs @@ -31,6 +31,13 @@ pub struct AgentStats { /// OTel JSONL filename written by Copilot CLI. pub const OTEL_FILENAME: &str = "otel.jsonl"; +/// Copilot CLI internal tool names excluded from the tool call count. +/// These are administrative spans, not user-visible tool invocations. +const INTERNAL_TOOL_NAMES: &[&str] = &[ + "execute_tool report_intent", + "execute_tool permission", +]; + impl AgentStats { /// Parse agent stats from an OTel JSONL file. /// @@ -47,7 +54,7 @@ impl AgentStats { /// /// Looks for: /// - The last `invoke_agent` span for aggregated tokens, model, turns, duration - /// - `execute_tool` spans for tool call count + /// - `execute_tool` spans for tool call count (excluding internal tools) pub fn from_otel_entries(entries: &[Value], agent_name: &str) -> Result { let mut stats = AgentStats { agent_name: agent_name.to_string(), @@ -97,14 +104,17 @@ impl AgentStats { stats.duration_seconds = compute_duration(span); } - // Count execute_tool spans + // Count execute_tool spans, excluding internal Copilot CLI tools stats.tool_calls = entries .iter() .filter(|e| { e.get("type").and_then(|t| t.as_str()) == Some("span") && e.get("name") .and_then(|n| n.as_str()) - .is_some_and(|n| n.starts_with("execute_tool")) + .is_some_and(|n| { + n.starts_with("execute_tool") + && !INTERNAL_TOOL_NAMES.contains(&n) + }) }) .count() as u64; @@ -117,9 +127,20 @@ impl AgentStats { } /// Render as a collapsible markdown stats block. + /// + /// `agent_name` and `model` are sanitized to remove control characters + /// and pipeline commands (`##vso[`), since the OTel file is writable + /// by the agent inside the AWF container. pub fn to_markdown(&self) -> String { let duration = format_duration(self.duration_seconds); - let model = self.model.as_deref().unwrap_or("unknown"); + + // Sanitize agent-controlled values before embedding in markdown. + // The OTel file lives in the AWF staging directory which the agent + // can write to, so model could be manipulated. + let model = sanitize_for_markdown( + self.model.as_deref().unwrap_or("unknown"), + ); + let name = sanitize_for_markdown(&self.agent_name); format!( "\n---\n\ @@ -135,7 +156,7 @@ impl AgentStats { | Turns | {turns} |\n\ \n\
\n", - name = self.agent_name, + name = name, model = model, input = format_number(self.input_tokens), output = format_number(self.output_tokens), @@ -147,6 +168,20 @@ impl AgentStats { } } +/// Sanitize a string for safe embedding in markdown output. +/// +/// Strips control characters and neutralizes ADO pipeline commands +/// (`##vso[`) that could be injected via the OTel file. +fn sanitize_for_markdown(s: &str) -> String { + s.chars() + .filter(|c| !c.is_control() || *c == '\n') + .collect::() + .replace("##vso[", "[vso-filtered][") + .replace("##[", "[filtered][") + // Strip pipe characters that could break markdown tables + .replace('|', "\\|") +} + /// Append agent stats markdown to a body string if stats are available /// and stats are not opted out. /// @@ -270,8 +305,8 @@ mod tests { assert_eq!(stats.output_tokens, 236); assert_eq!(stats.total_tokens(), 33185); assert_eq!(stats.turns, 2); - // execute_tool spans: report_intent + bash = 2 - assert_eq!(stats.tool_calls, 2); + // execute_tool spans: bash only (report_intent is filtered as internal) + assert_eq!(stats.tool_calls, 1); // Duration should be ~8 seconds (from the last invoke_agent span) assert!(stats.duration_seconds > 7.0 && stats.duration_seconds < 10.0); } @@ -316,4 +351,32 @@ mod tests { let d = compute_duration(&span); assert!((d - 8.282631).abs() < 0.01); } + + #[test] + fn test_sanitize_for_markdown_strips_vso_commands() { + assert_eq!( + sanitize_for_markdown("normal text"), + "normal text" + ); + assert_eq!( + sanitize_for_markdown("##vso[task.setvariable]evil"), + "[vso-filtered][task.setvariable]evil" + ); + assert_eq!( + sanitize_for_markdown("model|name"), + "model\\|name" + ); + } + + #[test] + fn test_internal_tools_excluded_from_count() { + let entries = vec![ + serde_json::json!({"type": "span", "name": "execute_tool report_intent"}), + serde_json::json!({"type": "span", "name": "execute_tool permission"}), + serde_json::json!({"type": "span", "name": "execute_tool bash"}), + serde_json::json!({"type": "span", "name": "execute_tool grep"}), + ]; + let stats = AgentStats::from_otel_entries(&entries, "test").unwrap(); + assert_eq!(stats.tool_calls, 2); // bash + grep, not report_intent or permission + } } diff --git a/src/execute.rs b/src/execute.rs index 303bb53e..80a8fc95 100644 --- a/src/execute.rs +++ b/src/execute.rs @@ -514,7 +514,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -548,7 +547,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -698,7 +696,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -742,7 +739,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -786,7 +782,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -835,7 +830,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -1043,7 +1037,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -1088,7 +1081,6 @@ mod tests { repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; diff --git a/src/main.rs b/src/main.rs index 12df306f..7f8c2204 100644 --- a/src/main.rs +++ b/src/main.rs @@ -216,7 +216,6 @@ async fn main() -> Result<()> { ctx.working_directory = safe_output_dir.clone(); ctx.tool_configs = front_matter.safe_outputs.clone(); ctx.allowed_repositories = allowed_repositories; - ctx.agent_name = Some(front_matter.name.clone()); // Load agent stats from OTel JSONL if available let otel_path = safe_output_dir.join(agent_stats::OTEL_FILENAME); diff --git a/src/safeoutputs/create_wiki_page.rs b/src/safeoutputs/create_wiki_page.rs index 354a2546..7cca151a 100644 --- a/src/safeoutputs/create_wiki_page.rs +++ b/src/safeoutputs/create_wiki_page.rs @@ -676,7 +676,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: std::collections::HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -741,7 +740,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -781,7 +779,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -821,7 +818,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; diff --git a/src/safeoutputs/result.rs b/src/safeoutputs/result.rs index d65e2a49..5a09ab14 100644 --- a/src/safeoutputs/result.rs +++ b/src/safeoutputs/result.rs @@ -56,8 +56,6 @@ pub struct ExecutionContext { /// Allowed repositories for PRs: "self" + checkout list aliases /// Maps alias to ADO repo name (e.g., "other-repo" -> "org/other-repo") pub allowed_repositories: HashMap, - /// Agent name from front matter (for display in stats blocks) - pub agent_name: Option, /// Agent execution statistics parsed from OTel JSONL pub agent_stats: Option, } @@ -113,7 +111,6 @@ impl Default for ExecutionContext { repository_id: std::env::var("BUILD_REPOSITORY_ID").ok(), repository_name: std::env::var("BUILD_REPOSITORY_NAME").ok(), allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, } } diff --git a/src/safeoutputs/update_wiki_page.rs b/src/safeoutputs/update_wiki_page.rs index 00d739ca..11f8cef0 100644 --- a/src/safeoutputs/update_wiki_page.rs +++ b/src/safeoutputs/update_wiki_page.rs @@ -648,7 +648,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: std::collections::HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -713,7 +712,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -753,7 +751,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -793,7 +790,6 @@ wiki-name: "MyProject.wiki" repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; diff --git a/src/safeoutputs/update_work_item.rs b/src/safeoutputs/update_work_item.rs index 94038e1c..59e2977b 100644 --- a/src/safeoutputs/update_work_item.rs +++ b/src/safeoutputs/update_work_item.rs @@ -716,7 +716,6 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -767,7 +766,6 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -814,7 +812,6 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; @@ -863,7 +860,6 @@ target: 42 repository_id: None, repository_name: None, allowed_repositories: HashMap::new(), - agent_name: None, agent_stats: None, }; From b3adefc07fd538deb7c784e258f2f5f1dbdb2b96 Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 22:52:46 +0100 Subject: [PATCH 5/8] fix: replace
tags with plain markdown for ADO compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Azure DevOps does not render HTML
/ as collapsible sections — they display as raw text. Switch to a horizontal rule + italic heading + table, which renders correctly across ADO PRs, work items, and wiki pages. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agent_stats.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/agent_stats.rs b/src/agent_stats.rs index 0b55bb26..ea4a73c0 100644 --- a/src/agent_stats.rs +++ b/src/agent_stats.rs @@ -126,7 +126,10 @@ impl AgentStats { self.input_tokens + self.output_tokens } - /// Render as a collapsible markdown stats block. + /// Render as a markdown stats block. + /// + /// Uses a horizontal rule and heading (not `
/`) + /// because Azure DevOps does not render HTML collapsible sections. /// /// `agent_name` and `model` are sanitized to remove control characters /// and pipeline commands (`##vso[`), since the OTel file is writable @@ -135,8 +138,6 @@ impl AgentStats { let duration = format_duration(self.duration_seconds); // Sanitize agent-controlled values before embedding in markdown. - // The OTel file lives in the AWF staging directory which the agent - // can write to, so model could be manipulated. let model = sanitize_for_markdown( self.model.as_deref().unwrap_or("unknown"), ); @@ -144,8 +145,7 @@ impl AgentStats { format!( "\n---\n\ -
\n\ - \u{1F916} Agent Stats ({name})\n\ + _\u{1F916} Agent Stats ({name})_\n\ \n\ | Metric | Value |\n\ |--------|-------|\n\ @@ -153,9 +153,7 @@ impl AgentStats { | Tokens | {input} input / {output} output ({total} total) |\n\ | Duration | {duration} |\n\ | Tool calls | {tools} |\n\ - | Turns | {turns} |\n\ - \n\ -
\n", + | Turns | {turns} |\n", name = name, model = model, input = format_number(self.input_tokens), @@ -323,7 +321,6 @@ mod tests { turns: 8, }; let md = stats.to_markdown(); - assert!(md.contains("
")); assert!(md.contains("Daily Code Review")); assert!(md.contains("claude-opus-4.5")); assert!(md.contains("45,230")); @@ -332,6 +329,7 @@ mod tests { assert!(md.contains("4m 32s")); assert!(md.contains("23")); assert!(md.contains("8")); + assert!(!md.contains("
"), "ADO does not support
tags"); } #[test] From 6751de78068af3290f0f775bf004c603b0fc12ff Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 22:57:02 +0100 Subject: [PATCH 6/8] refactor: compact single-line stats format, drop turns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace table with a single line using middle-dot separators: 🤖 _Agent Name_ · model · 45,230 in / 12,450 out · 23 tool calls · 4m 32s Remove turns from output (low value to operators). Remove unused total_tokens() method. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agent_stats.rs | 46 +++++++++++++--------------------------------- 1 file changed, 13 insertions(+), 33 deletions(-) diff --git a/src/agent_stats.rs b/src/agent_stats.rs index ea4a73c0..5fb307e1 100644 --- a/src/agent_stats.rs +++ b/src/agent_stats.rs @@ -121,23 +121,12 @@ impl AgentStats { Ok(stats) } - /// Total tokens (input + output). - pub fn total_tokens(&self) -> u64 { - self.input_tokens + self.output_tokens - } - - /// Render as a markdown stats block. - /// - /// Uses a horizontal rule and heading (not `
/`) - /// because Azure DevOps does not render HTML collapsible sections. + /// Render as a compact markdown stats line. /// - /// `agent_name` and `model` are sanitized to remove control characters - /// and pipeline commands (`##vso[`), since the OTel file is writable - /// by the agent inside the AWF container. + /// Uses middle-dot separators for a lightweight single-line format + /// that works across all ADO markdown surfaces. pub fn to_markdown(&self) -> String { let duration = format_duration(self.duration_seconds); - - // Sanitize agent-controlled values before embedding in markdown. let model = sanitize_for_markdown( self.model.as_deref().unwrap_or("unknown"), ); @@ -145,23 +134,15 @@ impl AgentStats { format!( "\n---\n\ - _\u{1F916} Agent Stats ({name})_\n\ - \n\ - | Metric | Value |\n\ - |--------|-------|\n\ - | Model | {model} |\n\ - | Tokens | {input} input / {output} output ({total} total) |\n\ - | Duration | {duration} |\n\ - | Tool calls | {tools} |\n\ - | Turns | {turns} |\n", + \u{1F916} _{name}_ \u{00B7} {model} \u{00B7} \ + {input} in / {output} out \u{00B7} \ + {tools} tool calls \u{00B7} {duration}\n", name = name, model = model, input = format_number(self.input_tokens), output = format_number(self.output_tokens), - total = format_number(self.total_tokens()), - duration = duration, tools = self.tool_calls, - turns = self.turns, + duration = duration, ) } } @@ -301,7 +282,7 @@ mod tests { assert_eq!(stats.model.as_deref(), Some("claude-sonnet-4.5")); assert_eq!(stats.input_tokens, 32949); assert_eq!(stats.output_tokens, 236); - assert_eq!(stats.total_tokens(), 33185); + assert_eq!(stats.input_tokens + stats.output_tokens, 33185); assert_eq!(stats.turns, 2); // execute_tool spans: bash only (report_intent is filtered as internal) assert_eq!(stats.tool_calls, 1); @@ -323,13 +304,12 @@ mod tests { let md = stats.to_markdown(); assert!(md.contains("Daily Code Review")); assert!(md.contains("claude-opus-4.5")); - assert!(md.contains("45,230")); - assert!(md.contains("12,450")); - assert!(md.contains("57,680")); + assert!(md.contains("45,230 in")); + assert!(md.contains("12,450 out")); + assert!(md.contains("23 tool calls")); assert!(md.contains("4m 32s")); - assert!(md.contains("23")); - assert!(md.contains("8")); - assert!(!md.contains("
"), "ADO does not support
tags"); + assert!(md.contains("\u{00B7}"), "should use middle-dot separators"); + assert!(!md.contains("turns"), "turns should not be in output"); } #[test] From 4cc92768e42a917a99b67b3befd00d02b7a8a6ba Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 23:09:27 +0100 Subject: [PATCH 7/8] fix: Default impl bugs, dead internal tool filter, sanitization, PR footer order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugs fixed: - CommentOnWorkItemConfig, CreateWikiPageConfig, UpdateWikiPageConfig: replace #[derive(Default)] with manual Default impls so include_stats defaults to true (not false from bool default) - Remove dead "execute_tool permission" from INTERNAL_TOOL_NAMES — the real OTel span is "permission" (no prefix), already excluded by the starts_with("execute_tool") predicate Improvements: - Strip newlines in sanitize_for_markdown (single-line format) - Reorder PR description: stats before provenance footer (footer is the final unambiguous security marker) - Add trailing newlines to add_pr_comment.rs and create_work_item.rs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agent_stats.rs | 17 +++++++++-------- src/safeoutputs/add_pr_comment.rs | 2 +- src/safeoutputs/comment_on_work_item.rs | 11 ++++++++++- src/safeoutputs/create_pr.rs | 9 +++++---- src/safeoutputs/create_wiki_page.rs | 16 +++++++++++++++- src/safeoutputs/create_work_item.rs | 2 +- src/safeoutputs/update_wiki_page.rs | 16 +++++++++++++++- 7 files changed, 56 insertions(+), 17 deletions(-) diff --git a/src/agent_stats.rs b/src/agent_stats.rs index 5fb307e1..2bfed7fd 100644 --- a/src/agent_stats.rs +++ b/src/agent_stats.rs @@ -33,9 +33,9 @@ pub const OTEL_FILENAME: &str = "otel.jsonl"; /// Copilot CLI internal tool names excluded from the tool call count. /// These are administrative spans, not user-visible tool invocations. +/// Names must include the "execute_tool " prefix as emitted in the OTel span name. const INTERNAL_TOOL_NAMES: &[&str] = &[ "execute_tool report_intent", - "execute_tool permission", ]; impl AgentStats { @@ -147,17 +147,17 @@ impl AgentStats { } } -/// Sanitize a string for safe embedding in markdown output. +/// Sanitize a string for safe embedding in a single-line markdown format. /// -/// Strips control characters and neutralizes ADO pipeline commands -/// (`##vso[`) that could be injected via the OTel file. +/// Strips control characters (including newlines — the stats line is +/// single-line), neutralizes ADO pipeline commands (`##vso[`), and +/// escapes pipe characters that break markdown tables. fn sanitize_for_markdown(s: &str) -> String { s.chars() - .filter(|c| !c.is_control() || *c == '\n') + .filter(|c| !c.is_control()) .collect::() .replace("##vso[", "[vso-filtered][") .replace("##[", "[filtered][") - // Strip pipe characters that could break markdown tables .replace('|', "\\|") } @@ -350,11 +350,12 @@ mod tests { fn test_internal_tools_excluded_from_count() { let entries = vec![ serde_json::json!({"type": "span", "name": "execute_tool report_intent"}), - serde_json::json!({"type": "span", "name": "execute_tool permission"}), serde_json::json!({"type": "span", "name": "execute_tool bash"}), serde_json::json!({"type": "span", "name": "execute_tool grep"}), + // "permission" span has no "execute_tool" prefix so is already excluded + serde_json::json!({"type": "span", "name": "permission"}), ]; let stats = AgentStats::from_otel_entries(&entries, "test").unwrap(); - assert_eq!(stats.tool_calls, 2); // bash + grep, not report_intent or permission + assert_eq!(stats.tool_calls, 2); // bash + grep only } } diff --git a/src/safeoutputs/add_pr_comment.rs b/src/safeoutputs/add_pr_comment.rs index 46577611..967043ab 100644 --- a/src/safeoutputs/add_pr_comment.rs +++ b/src/safeoutputs/add_pr_comment.rs @@ -624,4 +624,4 @@ allowed-statuses: fn default_include_stats() -> bool { true -} \ No newline at end of file +} diff --git a/src/safeoutputs/comment_on_work_item.rs b/src/safeoutputs/comment_on_work_item.rs index a473acc4..f692aba2 100644 --- a/src/safeoutputs/comment_on_work_item.rs +++ b/src/safeoutputs/comment_on_work_item.rs @@ -95,7 +95,7 @@ impl CommentTarget { /// max: 5 /// target: "*" /// ``` -#[derive(Debug, Clone, SanitizeConfig, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, SanitizeConfig, Serialize, Deserialize)] pub struct CommentOnWorkItemConfig { /// Target scope — which work items can be commented on. /// `None` means no target was configured; execution must reject this. @@ -106,6 +106,15 @@ pub struct CommentOnWorkItemConfig { pub include_stats: bool, } +impl Default for CommentOnWorkItemConfig { + fn default() -> Self { + Self { + target: None, + include_stats: true, + } + } +} + /// Fetch a work item's area path from the ADO API async fn get_work_item_area_path( client: &reqwest::Client, diff --git a/src/safeoutputs/create_pr.rs b/src/safeoutputs/create_pr.rs index 02a6db77..3a4e0034 100644 --- a/src/safeoutputs/create_pr.rs +++ b/src/safeoutputs/create_pr.rs @@ -1260,13 +1260,14 @@ impl Executor for CreatePrResult { } debug!("Changes pushed successfully"); - // Append provenance footer and agent stats to description - let description_with_footer = format!("{}{}", self.description, generate_pr_footer()); + // Append agent stats then provenance footer to description. + // Footer goes last as the final unambiguous provenance marker. let description_with_stats = crate::agent_stats::append_stats_to_body( - &description_with_footer, + &self.description, ctx, config.include_stats, ); + let description_final = format!("{}{}", description_with_stats, generate_pr_footer()); // Create the pull request via REST API info!("Creating pull request"); @@ -1280,7 +1281,7 @@ impl Executor for CreatePrResult { "sourceRefName": source_ref, "targetRefName": target_ref, "title": effective_title, - "description": description_with_stats, + "description": description_final, "isDraft": config.draft, }); diff --git a/src/safeoutputs/create_wiki_page.rs b/src/safeoutputs/create_wiki_page.rs index 7cca151a..74fdbf76 100644 --- a/src/safeoutputs/create_wiki_page.rs +++ b/src/safeoutputs/create_wiki_page.rs @@ -97,7 +97,7 @@ impl SanitizeContent for CreateWikiPageResult { /// title-prefix: "[Agent] " /// comment: "Created by agent" /// ``` -#[derive(Debug, Clone, SanitizeConfig, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, SanitizeConfig, Serialize, Deserialize)] pub struct CreateWikiPageConfig { /// Wiki identifier (name or ID). Required — execution fails without this. /// @@ -137,6 +137,20 @@ pub struct CreateWikiPageConfig { #[serde(default = "default_include_stats", rename = "include-stats")] pub include_stats: bool, } + +impl Default for CreateWikiPageConfig { + fn default() -> Self { + Self { + wiki_name: None, + wiki_project: None, + branch: None, + path_prefix: None, + title_prefix: None, + comment: None, + include_stats: true, + } + } +} // Path helpers // ============================================================================ diff --git a/src/safeoutputs/create_work_item.rs b/src/safeoutputs/create_work_item.rs index d8a4f4db..77794b91 100644 --- a/src/safeoutputs/create_work_item.rs +++ b/src/safeoutputs/create_work_item.rs @@ -537,4 +537,4 @@ tags: fn default_include_stats() -> bool { true -} \ No newline at end of file +} diff --git a/src/safeoutputs/update_wiki_page.rs b/src/safeoutputs/update_wiki_page.rs index 11f8cef0..98d5c37d 100644 --- a/src/safeoutputs/update_wiki_page.rs +++ b/src/safeoutputs/update_wiki_page.rs @@ -93,7 +93,7 @@ impl SanitizeContent for UpdateWikiPageResult { /// title-prefix: "[Agent] " /// comment: "Updated by agent" /// ``` -#[derive(Debug, Clone, SanitizeConfig, Default, Serialize, Deserialize)] +#[derive(Debug, Clone, SanitizeConfig, Serialize, Deserialize)] pub struct UpdateWikiPageConfig { /// Wiki identifier (name or ID). Required — execution fails without this. /// @@ -134,6 +134,20 @@ pub struct UpdateWikiPageConfig { pub include_stats: bool, } +impl Default for UpdateWikiPageConfig { + fn default() -> Self { + Self { + wiki_name: None, + wiki_project: None, + branch: None, + path_prefix: None, + title_prefix: None, + comment: None, + include_stats: true, + } + } +} + // ============================================================================ // Path helpers // ============================================================================ From 1236fb86ac3e2cb224318b16b7d03e574d74943a Mon Sep 17 00:00:00 2001 From: James Devine Date: Wed, 15 Apr 2026 23:21:31 +0100 Subject: [PATCH 8/8] fix: consolidate default_include_stats, plain agent name, doc comments, tests - Remove italic underscores from agent name in stats line (plain text) - Fix orphaned doc comment on compute_duration - Consolidate 5 duplicate default_include_stats() fns into single pub(crate) fn in agent_stats.rs - Add 3 unit tests for append_stats_to_body (opt-out, no-stats, with-stats) - Fix trailing newlines Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/agent_stats.rs | 50 ++++++++++++++++++++++++- src/safeoutputs/add_pr_comment.rs | 5 +-- src/safeoutputs/comment_on_work_item.rs | 5 +-- src/safeoutputs/create_wiki_page.rs | 5 +-- src/safeoutputs/create_work_item.rs | 5 +-- src/safeoutputs/update_wiki_page.rs | 5 +-- 6 files changed, 54 insertions(+), 21 deletions(-) diff --git a/src/agent_stats.rs b/src/agent_stats.rs index 2bfed7fd..686fe848 100644 --- a/src/agent_stats.rs +++ b/src/agent_stats.rs @@ -134,7 +134,7 @@ impl AgentStats { format!( "\n---\n\ - \u{1F916} _{name}_ \u{00B7} {model} \u{00B7} \ + \u{1F916} {name} \u{00B7} {model} \u{00B7} \ {input} in / {output} out \u{00B7} \ {tools} tool calls \u{00B7} {duration}\n", name = name, @@ -147,6 +147,13 @@ impl AgentStats { } } +/// Default value for `include_stats` serde fields (true). +/// +/// Used by safe output config structs via `#[serde(default = "...")]`. +pub(crate) fn default_include_stats() -> bool { + true +} + /// Sanitize a string for safe embedding in a single-line markdown format. /// /// Strips control characters (including newlines — the stats line is @@ -180,6 +187,8 @@ pub fn append_stats_to_body( None => body.to_string(), } } + +/// Compute the wall-clock duration of a span in seconds. /// /// Times are `[seconds, nanoseconds]` arrays. fn compute_duration(span: &Value) -> f64 { @@ -358,4 +367,43 @@ mod tests { let stats = AgentStats::from_otel_entries(&entries, "test").unwrap(); assert_eq!(stats.tool_calls, 2); // bash + grep only } + + #[test] + fn test_append_stats_to_body_opt_out() { + let mut ctx = crate::safeoutputs::ExecutionContext::default(); + ctx.agent_stats = Some(AgentStats { + agent_name: "test".to_string(), + model: Some("model".to_string()), + input_tokens: 100, + output_tokens: 50, + duration_seconds: 10.0, + tool_calls: 1, + turns: 1, + }); + assert_eq!(append_stats_to_body("body", &ctx, false), "body"); + } + + #[test] + fn test_append_stats_to_body_no_stats() { + let ctx = crate::safeoutputs::ExecutionContext::default(); // agent_stats: None + assert_eq!(append_stats_to_body("body", &ctx, true), "body"); + } + + #[test] + fn test_append_stats_to_body_with_stats() { + let mut ctx = crate::safeoutputs::ExecutionContext::default(); + ctx.agent_stats = Some(AgentStats { + agent_name: "test".to_string(), + model: Some("model".to_string()), + input_tokens: 100, + output_tokens: 50, + duration_seconds: 10.0, + tool_calls: 1, + turns: 1, + }); + let result = append_stats_to_body("body", &ctx, true); + assert!(result.starts_with("body")); + assert!(result.contains("test")); + assert!(result.contains("model")); + } } diff --git a/src/safeoutputs/add_pr_comment.rs b/src/safeoutputs/add_pr_comment.rs index 967043ab..e45f4ae0 100644 --- a/src/safeoutputs/add_pr_comment.rs +++ b/src/safeoutputs/add_pr_comment.rs @@ -151,7 +151,7 @@ pub struct AddPrCommentConfig { #[serde(default, rename = "allowed-statuses")] pub allowed_statuses: Vec, /// Whether to include agent execution stats in the output (default: true). - #[serde(default = "default_include_stats", rename = "include-stats")] + #[serde(default = "crate::agent_stats::default_include_stats", rename = "include-stats")] pub include_stats: bool, } @@ -622,6 +622,3 @@ allowed-statuses: } } -fn default_include_stats() -> bool { - true -} diff --git a/src/safeoutputs/comment_on_work_item.rs b/src/safeoutputs/comment_on_work_item.rs index f692aba2..f9263a40 100644 --- a/src/safeoutputs/comment_on_work_item.rs +++ b/src/safeoutputs/comment_on_work_item.rs @@ -102,7 +102,7 @@ pub struct CommentOnWorkItemConfig { pub target: Option, /// Whether to include agent execution stats in the comment (default: true). - #[serde(default = "default_include_stats", rename = "include-stats")] + #[serde(default = "crate::agent_stats::default_include_stats", rename = "include-stats")] pub include_stats: bool, } @@ -491,6 +491,3 @@ target: "*" } } -fn default_include_stats() -> bool { - true -} diff --git a/src/safeoutputs/create_wiki_page.rs b/src/safeoutputs/create_wiki_page.rs index 74fdbf76..d1e8cd59 100644 --- a/src/safeoutputs/create_wiki_page.rs +++ b/src/safeoutputs/create_wiki_page.rs @@ -134,7 +134,7 @@ pub struct CreateWikiPageConfig { pub comment: Option, /// Whether to include agent execution stats in the output (default: true). - #[serde(default = "default_include_stats", rename = "include-stats")] + #[serde(default = "crate::agent_stats::default_include_stats", rename = "include-stats")] pub include_stats: bool, } @@ -915,6 +915,3 @@ wiki-name: "MyProject.wiki" } } -fn default_include_stats() -> bool { - true -} \ No newline at end of file diff --git a/src/safeoutputs/create_work_item.rs b/src/safeoutputs/create_work_item.rs index 77794b91..b14be4b2 100644 --- a/src/safeoutputs/create_work_item.rs +++ b/src/safeoutputs/create_work_item.rs @@ -100,7 +100,7 @@ pub struct CreateWorkItemConfig { pub artifact_link: ArtifactLinkConfig, /// Whether to include agent execution stats in the output (default: true). - #[serde(default = "default_include_stats", rename = "include-stats")] + #[serde(default = "crate::agent_stats::default_include_stats", rename = "include-stats")] pub include_stats: bool, } @@ -535,6 +535,3 @@ tags: } } -fn default_include_stats() -> bool { - true -} diff --git a/src/safeoutputs/update_wiki_page.rs b/src/safeoutputs/update_wiki_page.rs index 98d5c37d..980a12c6 100644 --- a/src/safeoutputs/update_wiki_page.rs +++ b/src/safeoutputs/update_wiki_page.rs @@ -130,7 +130,7 @@ pub struct UpdateWikiPageConfig { pub comment: Option, /// Whether to include agent execution stats in the output (default: true). - #[serde(default = "default_include_stats", rename = "include-stats")] + #[serde(default = "crate::agent_stats::default_include_stats", rename = "include-stats")] pub include_stats: bool, } @@ -878,6 +878,3 @@ wiki-name: "MyProject.wiki" } } -fn default_include_stats() -> bool { - true -} \ No newline at end of file