|
| 1 | +//! Compile-time benchmark harness for vespera's proc-macros. |
| 2 | +//! |
| 3 | +//! Measures the `macro_expand_crate` rustc pass of a target fixture crate, |
| 4 | +//! which isolates the cost of expanding `vespera!`, `schema_type!`, and |
| 5 | +//! `#[derive(Schema)]` from the rest of compilation (type-check, codegen, |
| 6 | +//! LTO). Runs on **stable** via `RUSTC_BOOTSTRAP=1` (no nightly required), so |
| 7 | +//! it works in CI. |
| 8 | +//! |
| 9 | +//! ```text |
| 10 | +//! cargo run -p compile-bench-runner --release -- [OPTIONS] |
| 11 | +//! --target <CRATE> crate to measure (default: macro-compile-bench) |
| 12 | +//! --pass <NAME> -Z time-passes pass name (default: macro_expand_crate) |
| 13 | +//! --runs <N> measured iterations (default: 8) |
| 14 | +//! --save-baseline <X> write samples to baselines/<X>.txt |
| 15 | +//! --baseline <X> compare this run against baselines/<X>.txt |
| 16 | +//! ``` |
| 17 | +//! |
| 18 | +//! Methodology: each iteration runs `cargo clean -p <target>` to force a full |
| 19 | +//! re-expansion, then `cargo rustc … -- -Z time-passes` and parses the pass |
| 20 | +//! time. Compile time has only *positive* noise (a busy machine only ever |
| 21 | +//! adds time), so `min` is the robust point estimate; `median`/`mean`/`sd` |
| 22 | +//! are reported too. Gross outliers (> 3x median, e.g. AV/FS hiccups) are |
| 23 | +//! dropped before stats. |
| 24 | +
|
| 25 | +use std::env; |
| 26 | +use std::fs; |
| 27 | +use std::path::PathBuf; |
| 28 | +use std::process::Command; |
| 29 | + |
| 30 | +struct Args { |
| 31 | + target: String, |
| 32 | + pass: String, |
| 33 | + runs: usize, |
| 34 | + save_baseline: Option<String>, |
| 35 | + baseline: Option<String>, |
| 36 | +} |
| 37 | + |
| 38 | +fn print_help() { |
| 39 | + eprint!( |
| 40 | + "compile-bench-runner — vespera proc-macro compile-time benchmark\n\n\ |
| 41 | + USAGE: cargo run -p compile-bench-runner --release -- [OPTIONS]\n\ |
| 42 | + --target <CRATE> crate to measure (default: macro-compile-bench)\n\ |
| 43 | + --pass <NAME> -Z time-passes pass (default: macro_expand_crate)\n\ |
| 44 | + --runs <N> measured iterations (default: 8)\n\ |
| 45 | + --save-baseline <X> save samples to baselines/<X>.txt\n\ |
| 46 | + --baseline <X> compare against baselines/<X>.txt\n\ |
| 47 | + -h, --help this help\n" |
| 48 | + ); |
| 49 | +} |
| 50 | + |
| 51 | +fn parse_args() -> Args { |
| 52 | + let mut a = Args { |
| 53 | + target: "macro-compile-bench".to_owned(), |
| 54 | + pass: "macro_expand_crate".to_owned(), |
| 55 | + runs: 8, |
| 56 | + save_baseline: None, |
| 57 | + baseline: None, |
| 58 | + }; |
| 59 | + let mut it = env::args().skip(1); |
| 60 | + while let Some(arg) = it.next() { |
| 61 | + let mut next = |flag: &str| it.next().unwrap_or_else(|| fatal(&format!("{flag} needs a value"))); |
| 62 | + match arg.as_str() { |
| 63 | + "--target" => a.target = next("--target"), |
| 64 | + "--pass" => a.pass = next("--pass"), |
| 65 | + "--runs" => { |
| 66 | + a.runs = next("--runs").parse().unwrap_or_else(|_| fatal("--runs must be an integer")); |
| 67 | + } |
| 68 | + "--save-baseline" => a.save_baseline = Some(next("--save-baseline")), |
| 69 | + "--baseline" => a.baseline = Some(next("--baseline")), |
| 70 | + "-h" | "--help" => { |
| 71 | + print_help(); |
| 72 | + std::process::exit(0); |
| 73 | + } |
| 74 | + other => fatal(&format!("unknown argument: {other} (try --help)")), |
| 75 | + } |
| 76 | + } |
| 77 | + if a.runs == 0 { |
| 78 | + fatal("--runs must be >= 1"); |
| 79 | + } |
| 80 | + a |
| 81 | +} |
| 82 | + |
| 83 | +fn fatal(msg: &str) -> ! { |
| 84 | + eprintln!("error: {msg}"); |
| 85 | + std::process::exit(2); |
| 86 | +} |
| 87 | + |
| 88 | +/// A `cargo` command pre-seeded with `RUSTC_BOOTSTRAP=1` so `-Z time-passes` |
| 89 | +/// is accepted on a stable toolchain. |
| 90 | +fn cargo() -> Command { |
| 91 | + let mut c = Command::new(env::var("CARGO").unwrap_or_else(|_| "cargo".to_owned())); |
| 92 | + c.env("RUSTC_BOOTSTRAP", "1"); |
| 93 | + c |
| 94 | +} |
| 95 | + |
| 96 | +/// Extract the seconds for `pass` from `-Z time-passes` stderr. |
| 97 | +/// Lines look like: `time: 0.090; rss: 24MB -> 36MB ( +12MB)\tmacro_expand_crate`. |
| 98 | +fn extract_pass_time(stderr: &str, pass: &str) -> Option<f64> { |
| 99 | + for line in stderr.lines() { |
| 100 | + let line = line.trim_end(); |
| 101 | + if line.contains("time:") && line.split_whitespace().next_back() == Some(pass) { |
| 102 | + let after = line.split("time:").nth(1)?; |
| 103 | + return after.split(';').next()?.trim().parse::<f64>().ok(); |
| 104 | + } |
| 105 | + } |
| 106 | + None |
| 107 | +} |
| 108 | + |
| 109 | +fn measure_once(target: &str, pass: &str) -> Option<f64> { |
| 110 | + // Force a full re-expansion of the fixture lib (deps stay built). |
| 111 | + let _ = cargo().args(["clean", "-p", target]).status(); |
| 112 | + let out = cargo() |
| 113 | + .args(["rustc", "--quiet", "-p", target, "--lib", "--", "-Z", "time-passes"]) |
| 114 | + .output() |
| 115 | + .ok()?; |
| 116 | + let stderr = String::from_utf8_lossy(&out.stderr); |
| 117 | + extract_pass_time(&stderr, pass) |
| 118 | +} |
| 119 | + |
| 120 | +fn median(sorted: &[f64]) -> f64 { |
| 121 | + let n = sorted.len(); |
| 122 | + if n == 0 { |
| 123 | + return f64::NAN; |
| 124 | + } |
| 125 | + if n % 2 == 1 { |
| 126 | + sorted[n / 2] |
| 127 | + } else { |
| 128 | + (sorted[n / 2 - 1] + sorted[n / 2]) / 2.0 |
| 129 | + } |
| 130 | +} |
| 131 | + |
| 132 | +fn mean(v: &[f64]) -> f64 { |
| 133 | + v.iter().sum::<f64>() / v.len() as f64 |
| 134 | +} |
| 135 | + |
| 136 | +fn stddev(v: &[f64], m: f64) -> f64 { |
| 137 | + if v.len() < 2 { |
| 138 | + return 0.0; |
| 139 | + } |
| 140 | + (v.iter().map(|x| (x - m).powi(2)).sum::<f64>() / (v.len() - 1) as f64).sqrt() |
| 141 | +} |
| 142 | + |
| 143 | +fn baselines_dir() -> PathBuf { |
| 144 | + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("baselines") |
| 145 | +} |
| 146 | + |
| 147 | +fn main() { |
| 148 | + let args = parse_args(); |
| 149 | + |
| 150 | + eprintln!("[warm] building `{}` (and deps) once...", args.target); |
| 151 | + let warm = cargo() |
| 152 | + .args(["build", "--quiet", "-p", &args.target, "--lib"]) |
| 153 | + .status(); |
| 154 | + if !matches!(warm, Ok(s) if s.success()) { |
| 155 | + fatal(&format!("warm build of `{}` failed", args.target)); |
| 156 | + } |
| 157 | + |
| 158 | + eprintln!( |
| 159 | + "[measure] {} runs of `{}` on `{}`", |
| 160 | + args.runs, args.pass, args.target |
| 161 | + ); |
| 162 | + let mut samples = Vec::new(); |
| 163 | + for i in 0..args.runs { |
| 164 | + match measure_once(&args.target, &args.pass) { |
| 165 | + Some(t) => { |
| 166 | + eprintln!(" run {:>2}: {t:.4}s", i + 1); |
| 167 | + samples.push(t); |
| 168 | + } |
| 169 | + None => eprintln!(" run {:>2}: pass `{}` not found in output", i + 1, args.pass), |
| 170 | + } |
| 171 | + } |
| 172 | + if samples.is_empty() { |
| 173 | + fatal("no samples collected (is the target a lib that uses vespera macros?)"); |
| 174 | + } |
| 175 | + |
| 176 | + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); |
| 177 | + let med0 = median(&samples); |
| 178 | + let clean: Vec<f64> = samples.iter().copied().filter(|&t| t <= med0 * 3.0).collect(); |
| 179 | + let clean = if clean.is_empty() { samples.clone() } else { clean }; |
| 180 | + |
| 181 | + let min = clean[0]; |
| 182 | + let med = median(&clean); |
| 183 | + let mn = mean(&clean); |
| 184 | + let sd = stddev(&clean, mn); |
| 185 | + let rel_sd = if mn > 0.0 { 100.0 * sd / mn } else { 0.0 }; |
| 186 | + |
| 187 | + println!(); |
| 188 | + println!( |
| 189 | + "== {} on `{}` ({} clean / {} total runs) ==", |
| 190 | + args.pass, |
| 191 | + args.target, |
| 192 | + clean.len(), |
| 193 | + samples.len() |
| 194 | + ); |
| 195 | + println!(" min={min:.4}s median={med:.4}s mean={mn:.4}s sd={sd:.4}s ({rel_sd:.1}%)"); |
| 196 | + |
| 197 | + if let Some(name) = &args.save_baseline { |
| 198 | + let dir = baselines_dir(); |
| 199 | + let _ = fs::create_dir_all(&dir); |
| 200 | + let path = dir.join(format!("{name}.txt")); |
| 201 | + let body: String = clean.iter().map(|t| format!("{t}\n")).collect(); |
| 202 | + match fs::write(&path, body) { |
| 203 | + Ok(()) => println!(" saved baseline `{name}` -> {}", path.display()), |
| 204 | + Err(e) => eprintln!(" failed to save baseline `{name}`: {e}"), |
| 205 | + } |
| 206 | + } |
| 207 | + |
| 208 | + if let Some(name) = &args.baseline { |
| 209 | + let path = baselines_dir().join(format!("{name}.txt")); |
| 210 | + match fs::read_to_string(&path) { |
| 211 | + Ok(s) => { |
| 212 | + let mut base: Vec<f64> = |
| 213 | + s.lines().filter_map(|l| l.trim().parse().ok()).collect(); |
| 214 | + if base.is_empty() { |
| 215 | + eprintln!(" baseline `{name}` is empty"); |
| 216 | + } else { |
| 217 | + base.sort_by(|a, b| a.partial_cmp(b).unwrap()); |
| 218 | + let bmin = base[0]; |
| 219 | + let bmed = median(&base); |
| 220 | + let d_min = 100.0 * (min - bmin) / bmin; |
| 221 | + let d_med = 100.0 * (med - bmed) / bmed; |
| 222 | + // Noise-aware verdict on `min` (the robust estimator): |
| 223 | + // require the change to exceed run-to-run noise (>= 2%). |
| 224 | + let noise = rel_sd.max(2.0); |
| 225 | + let verdict = if d_min.abs() <= noise { |
| 226 | + "NO CHANGE (within noise)" |
| 227 | + } else if d_min < 0.0 { |
| 228 | + "IMPROVED" |
| 229 | + } else { |
| 230 | + "REGRESSED" |
| 231 | + }; |
| 232 | + println!(); |
| 233 | + println!("== vs baseline `{name}` =="); |
| 234 | + println!(" min: {bmin:.4}s -> {min:.4}s ({d_min:+.1}%)"); |
| 235 | + println!(" median: {bmed:.4}s -> {med:.4}s ({d_med:+.1}%)"); |
| 236 | + println!(" verdict: {verdict} (noise ~{noise:.1}%)"); |
| 237 | + } |
| 238 | + } |
| 239 | + Err(e) => eprintln!(" baseline `{name}` not found ({e}); use --save-baseline first"), |
| 240 | + } |
| 241 | + } |
| 242 | +} |
0 commit comments