Skip to content

Commit 182ef5e

Browse files
prontclaude
andcommitted
feat: add automated-review-stats command
Adds a new CLI command to measure how Vector contributors react to chatgpt-codex-connector review comments (liked / disliked / no reaction). Includes a discovery mode (omit --bot-login) that lists all review comment authors by frequency, used to identify the correct bot login. On each run the command writes: - out/automated-review-stats/{owner}_{repo}.csv — full per-comment table - trends/{repo}.md — two summary tables (all reactions + reacted-only) updated in place via AUTO: markers Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 9e0918e commit 182ef5e

4 files changed

Lines changed: 318 additions & 2 deletions

File tree

Lines changed: 275 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,275 @@
1+
use crate::commands::fetch_issues::parse_since;
2+
use crate::config::Config;
3+
use anyhow::{Context, Result};
4+
use reqwest::blocking::Client;
5+
use serde_json::{json, Value};
6+
use std::collections::HashMap;
7+
use std::fs;
8+
9+
const GRAPHQL_URL: &str = "https://api.github.com/graphql";
10+
const PR_PAGE_SIZE: u32 = 10;
11+
12+
const MERGED_PRS_QUERY: &str = r#"
13+
query($owner: String!, $name: String!, $first: Int!, $after: String) {
14+
repository(owner: $owner, name: $name) {
15+
pullRequests(first: $first, after: $after, states: [MERGED], orderBy: {field: UPDATED_AT, direction: DESC}) {
16+
pageInfo {
17+
endCursor
18+
hasNextPage
19+
}
20+
nodes {
21+
number
22+
mergedAt
23+
reviewThreads(first: 30) {
24+
nodes {
25+
comments(first: 10) {
26+
nodes {
27+
url
28+
author { login }
29+
reactions(first: 20) {
30+
nodes {
31+
content
32+
}
33+
}
34+
}
35+
}
36+
}
37+
}
38+
}
39+
}
40+
}
41+
}
42+
"#;
43+
44+
struct Stats {
45+
prs_scanned: u64,
46+
total: u64,
47+
liked: u64,
48+
disliked: u64,
49+
no_reaction: u64,
50+
}
51+
52+
/// Omit `bot_login` to run in discovery mode: lists all review comment authors.
53+
pub fn run(config: &Config, bot_login: Option<&str>, since: Option<&str>) -> Result<()> {
54+
let client = Client::new();
55+
let since_ts = since.map(parse_since).transpose()?;
56+
57+
let discovery = bot_login.is_none();
58+
let mut authors: HashMap<String, u64> = HashMap::new();
59+
let mut stats = Stats { prs_scanned: 0, total: 0, liked: 0, disliked: 0, no_reaction: 0 };
60+
let mut rows: Vec<(String, &'static str)> = Vec::new();
61+
let mut after: Option<String> = None;
62+
let mut page = 1;
63+
64+
loop {
65+
println!("Fetching merged PR page {page} (batch: {PR_PAGE_SIZE})...");
66+
67+
let body = json!({
68+
"query": MERGED_PRS_QUERY,
69+
"variables": {
70+
"owner": config.repo_owner,
71+
"name": config.repo_name,
72+
"first": PR_PAGE_SIZE,
73+
"after": after,
74+
}
75+
});
76+
77+
let response = client
78+
.post(GRAPHQL_URL)
79+
.header("Authorization", format!("Bearer {}", config.github_token))
80+
.header("Accept", "application/vnd.github+json")
81+
.header("User-Agent", "github-tools")
82+
.json(&body)
83+
.send()
84+
.context("Failed to send GraphQL request")?;
85+
86+
if !response.status().is_success() {
87+
let status = response.status();
88+
let text = response.text().unwrap_or_default();
89+
anyhow::bail!("GraphQL request failed ({status}): {text}");
90+
}
91+
92+
let result: Value = response.json().context("Failed to parse GraphQL response")?;
93+
94+
if let Some(errors) = result.get("errors") {
95+
anyhow::bail!("GraphQL errors: {errors}");
96+
}
97+
98+
let connection = &result["data"]["repository"]["pullRequests"];
99+
let nodes = connection["nodes"].as_array().context("Missing nodes")?;
100+
let has_next = connection["pageInfo"]["hasNextPage"].as_bool().unwrap_or(false);
101+
let end_cursor = connection["pageInfo"]["endCursor"].as_str().map(|s| s.to_string());
102+
103+
let mut hit_boundary = false;
104+
for pr in nodes {
105+
let merged_at = pr["mergedAt"].as_str().unwrap_or("");
106+
if let Some(ref ts) = since_ts {
107+
if merged_at < ts.as_str() {
108+
hit_boundary = true;
109+
continue;
110+
}
111+
}
112+
stats.prs_scanned += 1;
113+
114+
let threads = pr["reviewThreads"]["nodes"].as_array().cloned().unwrap_or_default();
115+
for thread in &threads {
116+
let comments = thread["comments"]["nodes"].as_array().cloned().unwrap_or_default();
117+
for comment in &comments {
118+
let author = comment["author"]["login"].as_str().unwrap_or("");
119+
120+
if discovery {
121+
*authors.entry(author.to_string()).or_default() += 1;
122+
continue;
123+
}
124+
125+
if Some(author) != bot_login {
126+
continue;
127+
}
128+
129+
stats.total += 1;
130+
let url = comment["url"].as_str().unwrap_or("").to_string();
131+
let reactions = comment["reactions"]["nodes"].as_array().cloned().unwrap_or_default();
132+
let has_up = reactions.iter().any(|r| r["content"].as_str() == Some("THUMBS_UP"));
133+
let has_down = reactions.iter().any(|r| r["content"].as_str() == Some("THUMBS_DOWN"));
134+
135+
let reaction = if has_up {
136+
stats.liked += 1;
137+
"liked"
138+
} else if has_down {
139+
stats.disliked += 1;
140+
"disliked"
141+
} else {
142+
stats.no_reaction += 1;
143+
"no reaction"
144+
};
145+
rows.push((url, reaction));
146+
}
147+
}
148+
}
149+
150+
println!(" PRs scanned: {} | bot comments found: {}", stats.prs_scanned, stats.total);
151+
152+
if hit_boundary || !has_next {
153+
break;
154+
}
155+
after = end_cursor;
156+
page += 1;
157+
}
158+
159+
println!();
160+
161+
if discovery {
162+
println!("=== Review Comment Authors ({}/{}) ===", config.repo_owner, config.repo_name);
163+
if let Some(ref ts) = since_ts {
164+
println!("Since : {ts}");
165+
}
166+
println!("PRs scanned : {}", stats.prs_scanned);
167+
println!();
168+
let mut sorted: Vec<_> = authors.iter().collect();
169+
sorted.sort_by(|a, b| b.1.cmp(a.1));
170+
for (login, count) in &sorted {
171+
println!(" {count:>6} {login}");
172+
}
173+
return Ok(());
174+
}
175+
176+
let login = bot_login.unwrap();
177+
println!("=== Automated Review Stats ({}/{}) ===", config.repo_owner, config.repo_name);
178+
println!("Bot login : {login}");
179+
if let Some(ref ts) = since_ts {
180+
println!("Since : {ts}");
181+
}
182+
println!("PRs scanned : {}", stats.prs_scanned);
183+
println!("Total bot comments : {}", stats.total);
184+
println!(" Liked (THUMBS_UP) : {}", stats.liked);
185+
println!(" Disliked (THUMBS_DOWN) : {}", stats.disliked);
186+
println!(" No reaction : {}", stats.no_reaction);
187+
if stats.total > 0 {
188+
println!();
189+
println!(" Like rate : {:.1}%", stats.liked as f64 / stats.total as f64 * 100.0);
190+
println!(" Dislike rate : {:.1}%", stats.disliked as f64 / stats.total as f64 * 100.0);
191+
}
192+
193+
if !rows.is_empty() {
194+
let out_dir = std::path::Path::new("out/automated-review-stats");
195+
fs::create_dir_all(out_dir)?;
196+
let csv_path = out_dir.join(format!("{}_{}.csv", config.repo_owner, config.repo_name));
197+
rows.sort_by_key(|(_, reaction)| match *reaction {
198+
"no reaction" => 0,
199+
"disliked" => 1,
200+
_ => 2,
201+
});
202+
let mut csv = String::from("url, reaction\n");
203+
for (url, reaction) in &rows {
204+
csv.push_str(&format!("{url}, {reaction}\n"));
205+
}
206+
fs::write(&csv_path, csv)?;
207+
println!();
208+
println!("Full table written to {}", csv_path.display());
209+
}
210+
211+
if stats.total > 0 {
212+
update_trends(config, login, since_ts.as_deref(), &stats)?;
213+
}
214+
215+
Ok(())
216+
}
217+
218+
fn update_trends(config: &Config, bot_login: &str, since_ts: Option<&str>, stats: &Stats) -> Result<()> {
219+
let trends_path = std::path::Path::new("trends").join(format!("{}.md", config.repo_name));
220+
if !trends_path.exists() {
221+
println!("Trends file not found at {}, skipping.", trends_path.display());
222+
return Ok(());
223+
}
224+
225+
let reacted = stats.liked + stats.disliked;
226+
let since_label = since_ts.unwrap_or("all time");
227+
228+
let new_content = format!(
229+
"\n**All comments** ({} merged PRs, bot: `{bot_login}`, since: {since_label})\n\
230+
\n\
231+
| Reaction | Count | Share |\n\
232+
|----------|------:|------:|\n\
233+
| Liked 👍 | {liked} | {liked_pct:.1}% |\n\
234+
| Disliked 👎 | {disliked} | {disliked_pct:.1}% |\n\
235+
| No reaction | {no_reaction} | {no_reaction_pct:.1}% |\n\
236+
| **Total** | **{total}** | |\n\
237+
\n\
238+
**Reacted comments only** (excludes no reaction)\n\
239+
\n\
240+
| Reaction | Count | Share |\n\
241+
|----------|------:|------:|\n\
242+
| Liked 👍 | {liked} | {liked_reacted_pct:.1}% |\n\
243+
| Disliked 👎 | {disliked} | {disliked_reacted_pct:.1}% |\n\
244+
| **Total** | **{reacted}** | |\n",
245+
stats.prs_scanned,
246+
liked = stats.liked,
247+
disliked = stats.disliked,
248+
no_reaction = stats.no_reaction,
249+
total = stats.total,
250+
liked_pct = stats.liked as f64 / stats.total as f64 * 100.0,
251+
disliked_pct = stats.disliked as f64 / stats.total as f64 * 100.0,
252+
no_reaction_pct = stats.no_reaction as f64 / stats.total as f64 * 100.0,
253+
liked_reacted_pct = stats.liked as f64 / reacted as f64 * 100.0,
254+
disliked_reacted_pct = stats.disliked as f64 / reacted as f64 * 100.0,
255+
);
256+
257+
const START: &str = "<!-- AUTO:automated-review-stats:start -->";
258+
const END: &str = "<!-- AUTO:automated-review-stats:end -->";
259+
260+
let existing = fs::read_to_string(&trends_path)?;
261+
262+
let updated = match (existing.find(START), existing.find(END)) {
263+
(Some(start_pos), Some(end_pos)) => {
264+
let before = &existing[..start_pos + START.len()];
265+
let after = &existing[end_pos..];
266+
format!("{before}{new_content}{after}")
267+
}
268+
_ => format!("{existing}\n## AI-Assisted Code Review\n\n{START}{new_content}{END}\n"),
269+
};
270+
271+
fs::write(&trends_path, updated)?;
272+
println!("Trends updated at {}", trends_path.display());
273+
274+
Ok(())
275+
}

src/commands/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ pub mod build_db;
22
pub mod compact;
33
pub mod close_old_prs;
44
pub mod delete_stale_branches;
5+
pub mod fetch_automated_review_stats;
56
pub mod fetch_discussions;
67
pub mod fetch_issues;
78
pub mod fetch_labels;

src/main.rs

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@ use anyhow::Result;
22
use clap::{Parser, Subcommand};
33
use github_tools::{
44
commands::{
5-
build_db, close_old_prs, compact, delete_stale_branches, fetch_discussions, fetch_issues,
6-
fetch_labels, generate_summaries, purge, remove_legacy_label, workflows,
5+
build_db, close_old_prs, compact, delete_stale_branches, fetch_automated_review_stats,
6+
fetch_discussions, fetch_issues, fetch_labels, generate_summaries, purge,
7+
remove_legacy_label, workflows,
78
},
89
config::Config,
910
};
@@ -121,6 +122,16 @@ enum Command {
121122
#[arg(long)]
122123
limit: Option<usize>,
123124
},
125+
/// Count automated review comments by reaction (liked / disliked / no reaction).
126+
/// Omit --bot-login to list all review comment authors and discover the right login.
127+
AutomatedReviewStats {
128+
#[arg(long, help = "Path to .env file")]
129+
env_file: Option<String>,
130+
#[arg(long, help = "GitHub login of the review bot; omit to list all authors")]
131+
bot_login: Option<String>,
132+
#[arg(long, help = "Only scan PRs merged since this date (ISO, YYYY-MM, or relative: 3m, 1y, 30d)")]
133+
since: Option<String>,
134+
},
124135
/// Deduplicate JSON year files in a data directory
125136
Compact {
126137
#[arg(help = "Path to directory containing year JSON files (e.g. data/vectordotdev_vector/issues)")]
@@ -190,6 +201,14 @@ fn main() -> Result<()> {
190201
let config = Config::load(env_file.as_deref())?;
191202
build_db::run(&input, &config)
192203
}
204+
Command::AutomatedReviewStats {
205+
env_file,
206+
bot_login,
207+
since,
208+
} => {
209+
let config = Config::load(env_file.as_deref())?;
210+
fetch_automated_review_stats::run(&config, bot_login.as_deref(), since.as_deref())
211+
}
193212
Command::Compact { dir } => compact::run(&dir),
194213
Command::FetchAll { env_file, since } => workflows::fetch_all(&env_file, since.as_deref()),
195214
Command::GenerateAll {

trends/vector.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,24 @@
9191
## Discussions
9292

9393
![Monthly Discussions](../data/images/vectordotdev_vector_discussions.monthly_trend.png)
94+
95+
## AI-Assisted Code Review
96+
97+
<!-- AUTO:automated-review-stats:start -->
98+
**All comments** (169 merged PRs, bot: `chatgpt-codex-connector`, since: 2026-01-01T00:00:00Z)
99+
100+
| Reaction | Count | Share |
101+
|----------|------:|------:|
102+
| Liked 👍 | 125 | 54.1% |
103+
| Disliked 👎 | 26 | 11.3% |
104+
| No reaction | 80 | 34.6% |
105+
| **Total** | **231** | |
106+
107+
**Reacted comments only** (excludes no reaction)
108+
109+
| Reaction | Count | Share |
110+
|----------|------:|------:|
111+
| Liked 👍 | 125 | 82.8% |
112+
| Disliked 👎 | 26 | 17.2% |
113+
| **Total** | **151** | |
114+
<!-- AUTO:automated-review-stats:end -->

0 commit comments

Comments
 (0)