Skip to content

Commit 73699bf

Browse files
hyperpolymathclaude
andcommitted
bench: add Six Sigma ops benchmark (Rust)
bench/bench_sigma_ops.rs — synthetic Tangle parse-throughput and IR construction benchmark with Six Sigma classification. Covers: - parse small/large program (lexer proxy until crate is linked) - lex throughput (ns/byte) - CrossingIR chain construction (1 000 nodes, sign ∈ {+1,-1,0}) - Vec<u8> allocation micro-benchmarks All baselines at 0.0 (baseline run); copy printed ns values into the BASELINES struct for CI comparisons. Exits with code 1 on UNACCEPTABLE regressions. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent b50c340 commit 73699bf

1 file changed

Lines changed: 301 additions & 0 deletions

File tree

bench/bench_sigma_ops.rs

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
// SPDX-License-Identifier: PMPL-1.0-or-later
2+
// (MPL-2.0 is automatic legal fallback until PMPL is formally recognised)
3+
//
4+
// bench_sigma_ops.rs — Tangle parser + IR ops benchmark with Six Sigma
5+
// classification.
6+
//
7+
// Measures parse throughput (LOC/sec) and IR construction time for a
8+
// synthetic Tangle program, then classifies each result against stored
9+
// baselines using the hyperpolymath Six Sigma taxonomy:
10+
//
11+
// UNACCEPTABLE : >50 % regression → hard fail
12+
// ACCEPTABLE : 20–50 % regression → soft fail
13+
// ORDINARY : ±20 % → pass
14+
// EXTRAORDINARY : >20 % improvement → pass + flag
15+
//
16+
// First run: BASELINES are 0.0 → every result is printed as "[BASELINE]".
17+
// Copy the printed values into BASELINES for subsequent CI comparisons.
18+
//
19+
// Usage (from tangle/src/rust/):
20+
// cargo run --release --example bench_sigma_ops
21+
// (file lives in tangle/bench/; symlink or copy to src/rust/examples/)
22+
23+
use std::time::{Duration, Instant};
24+
25+
// ── Six Sigma baselines (nanoseconds) ─────────────────────────────────────────
26+
// Populate from a "[BASELINE]" run. 0.0 means "unset → baseline run".
27+
struct Baselines {
28+
parse_small_ns: f64, // 10-def program, 100 iterations
29+
parse_large_ns: f64, // 40-def program, 100 iterations
30+
lex_throughput_ns: f64, // ns per source byte for large program
31+
ir_construct_ns: f64, // synthetic CrossingIR chain, 1 000 nodes
32+
alloc_vec_small_ns: f64, // Vec<u8> push 100 bytes
33+
alloc_vec_large_ns: f64, // Vec<u8> push 10 000 bytes
34+
}
35+
36+
const BASELINES: Baselines = Baselines {
37+
parse_small_ns: 0.0,
38+
parse_large_ns: 0.0,
39+
lex_throughput_ns: 0.0,
40+
ir_construct_ns: 0.0,
41+
alloc_vec_small_ns: 0.0,
42+
alloc_vec_large_ns: 0.0,
43+
};
44+
45+
// ── Six Sigma classifier ───────────────────────────────────────────────────────
46+
#[derive(Debug, PartialEq)]
47+
enum SigmaTier { Baseline, Extraordinary, Ordinary, Acceptable, Unacceptable }
48+
49+
struct SigmaSummary {
50+
baseline: usize,
51+
extraordinary: usize,
52+
ordinary: usize,
53+
acceptable: usize,
54+
unacceptable: usize,
55+
}
56+
57+
impl SigmaSummary {
58+
fn new() -> Self {
59+
Self { baseline: 0, extraordinary: 0, ordinary: 0, acceptable: 0, unacceptable: 0 }
60+
}
61+
62+
fn classify(&mut self, label: &str, measured_ns: f64, baseline_ns: f64) -> SigmaTier {
63+
if baseline_ns == 0.0 {
64+
println!(" [BASELINE] {:<42} {:.1} ns", label, measured_ns);
65+
self.baseline += 1;
66+
return SigmaTier::Baseline;
67+
}
68+
let pct = (measured_ns - baseline_ns) / baseline_ns * 100.0;
69+
if pct > 50.0 {
70+
println!(" [UNACCEPTABLE] {:<42} {:+.1} % HARD FAIL", label, pct);
71+
self.unacceptable += 1;
72+
SigmaTier::Unacceptable
73+
} else if pct > 20.0 {
74+
println!(" [ACCEPTABLE] {:<42} {:+.1} % soft fail", label, pct);
75+
self.acceptable += 1;
76+
SigmaTier::Acceptable
77+
} else if pct >= -20.0 {
78+
println!(" [ORDINARY] {:<42} {:+.1} %", label, pct);
79+
self.ordinary += 1;
80+
SigmaTier::Ordinary
81+
} else {
82+
println!(" [EXTRAORDINARY] {:<42} {:+.1} % improvement", label, pct);
83+
self.extraordinary += 1;
84+
SigmaTier::Extraordinary
85+
}
86+
}
87+
}
88+
89+
// ── Timing helpers ─────────────────────────────────────────────────────────────
90+
fn bench_ns<F: FnMut()>(mut f: F, warmup: usize, iterations: usize) -> f64 {
91+
for _ in 0..warmup { f(); }
92+
let mut times = Vec::with_capacity(iterations);
93+
for _ in 0..iterations {
94+
let t0 = Instant::now();
95+
f();
96+
times.push(t0.elapsed().as_nanos() as f64);
97+
}
98+
times.sort_by(f64::total_cmp);
99+
times[iterations / 2] // median
100+
}
101+
102+
fn ns_per_byte(total_ns: f64, bytes: usize, iterations: usize) -> f64 {
103+
total_ns / (bytes * iterations) as f64
104+
}
105+
106+
// ── Synthetic Tangle program generators ───────────────────────────────────────
107+
fn generate_program(num_defs: usize) -> String {
108+
let mut buf = String::with_capacity(num_defs * 400);
109+
for i in 0..num_defs {
110+
buf.push_str(&format!("def knot_{i}(a, b) = a . b + b . a\n\n"));
111+
buf.push_str(&format!("def compose_{i}(x, y) = x . y | y . x\n\n"));
112+
if i % 5 == 0 {
113+
buf.push_str(&format!(
114+
"weave strands s{i}: wire into\n s{i} . s{i}\nyield strands out{i}\n\n"
115+
));
116+
}
117+
if i % 8 == 0 {
118+
buf.push_str(&format!("compute jones(knot_{i}(1, 2))\n\n"));
119+
}
120+
if i % 7 == 0 {
121+
buf.push_str(&format!("def classify_{i}(x) =\n"));
122+
buf.push_str(" match x with\n | identity -> 0\n");
123+
buf.push_str(&format!(" | y -> {}\n end\n\n", i + 1));
124+
}
125+
if i % 6 == 0 {
126+
buf.push_str(&format!("def bind_{i} = let t = {i} + {} in t * 2\n\n", i + 1));
127+
}
128+
if i % 10 == 0 {
129+
buf.push_str(&format!("assert knot_{i}(1, 2) == knot_{i}(1, 2)\n\n"));
130+
}
131+
}
132+
buf
133+
}
134+
135+
// ── Synthetic IR construction (CrossingIR chain) ───────────────────────────────
136+
// CrossingIR: (id: usize, sign: i8, arcs: (usize, usize, usize, usize))
137+
// sign ∈ {+1, -1, 0} where 0 = virtual crossing (Kauffman 1999).
138+
#[derive(Debug)]
139+
struct CrossingIR {
140+
id: usize,
141+
sign: i8,
142+
arcs: (usize, usize, usize, usize),
143+
}
144+
145+
fn build_crossing_chain(n: usize) -> Vec<CrossingIR> {
146+
let mut chain = Vec::with_capacity(n);
147+
for i in 0..n {
148+
let sign: i8 = match i % 3 {
149+
0 => 1, // positive crossing
150+
1 => -1, // negative crossing
151+
_ => 0, // virtual crossing
152+
};
153+
chain.push(CrossingIR {
154+
id: i,
155+
sign,
156+
arcs: (2 * i, 2 * i + 1, 2 * i + 2, 2 * i + 3),
157+
});
158+
}
159+
std::hint::black_box(chain)
160+
}
161+
162+
// ── Main ───────────────────────────────────────────────────────────────────────
163+
fn main() {
164+
let mut sigma = SigmaSummary::new();
165+
166+
let small_src = generate_program(10);
167+
let large_src = generate_program(40);
168+
let small_loc = small_src.lines().count();
169+
let large_loc = large_src.lines().count();
170+
let large_bytes = large_src.len();
171+
172+
println!("=== Tangle Benchmark Suite (Six Sigma) ===");
173+
println!("Small program: {} LOC, {} bytes", small_loc, small_src.len());
174+
println!("Large program: {} LOC, {} bytes\n", large_loc, large_bytes);
175+
176+
// ── Parse throughput (using raw string scanning as proxy until parser is wired) ──
177+
// The tangle parser (Rust frontend) is called via tangle::parser::Parser
178+
// when compiled as part of the crate. These benchmarks measure the
179+
// cost of the parse *pipeline* (tokenisation + AST construction) using
180+
// the source-level string as input. Replace the scan_tokens stub with
181+
// `tangle::lexer::Lexer::tokenize` once the crate is linked.
182+
println!("─── Parse throughput (lexer proxy) ──────────────────────────────────────");
183+
184+
let small_parse_ns = bench_ns(
185+
|| { let _ = std::hint::black_box(scan_tokens(&small_src)); },
186+
3, 100,
187+
);
188+
sigma.classify("parse small (10 defs, 100 iters)", small_parse_ns, BASELINES.parse_small_ns);
189+
190+
let large_parse_ns = bench_ns(
191+
|| { let _ = std::hint::black_box(scan_tokens(&large_src)); },
192+
3, 100,
193+
);
194+
sigma.classify("parse large (40 defs, 100 iters)", large_parse_ns, BASELINES.parse_large_ns);
195+
196+
let lex_byte_ns = ns_per_byte(large_parse_ns, large_bytes, 1);
197+
sigma.classify("lex throughput (ns/byte, large)", lex_byte_ns, BASELINES.lex_throughput_ns);
198+
199+
println!();
200+
201+
// ── IR construction ─────────────────────────────────────────────────────────
202+
println!("─── IR construction ─────────────────────────────────────────────────────");
203+
204+
let ir_ns = bench_ns(
205+
|| { let _ = build_crossing_chain(1_000); },
206+
3, 100,
207+
);
208+
sigma.classify("CrossingIR chain (1 000 nodes)", ir_ns, BASELINES.ir_construct_ns);
209+
210+
println!();
211+
212+
// ── Allocation micro-benchmarks ─────────────────────────────────────────────
213+
println!("─── Allocation micro-benchmarks ─────────────────────────────────────────");
214+
215+
let alloc_small_ns = bench_ns(
216+
|| {
217+
let mut v: Vec<u8> = Vec::with_capacity(100);
218+
for b in 0u8..100 { v.push(b); }
219+
std::hint::black_box(v);
220+
},
221+
3, 1_000,
222+
);
223+
sigma.classify("Vec<u8> push 100 bytes", alloc_small_ns, BASELINES.alloc_vec_small_ns);
224+
225+
let alloc_large_ns = bench_ns(
226+
|| {
227+
let mut v: Vec<u8> = Vec::with_capacity(10_000);
228+
for b in 0u8..=255 {
229+
for _ in 0..(10_000 / 256) { v.push(b); }
230+
}
231+
std::hint::black_box(v);
232+
},
233+
3, 200,
234+
);
235+
sigma.classify("Vec<u8> push 10 000 bytes", alloc_large_ns, BASELINES.alloc_vec_large_ns);
236+
237+
println!();
238+
239+
// ── Summary ─────────────────────────────────────────────────────────────────
240+
println!("─── Six Sigma Summary ───────────────────────────────────────────────────");
241+
let total = sigma.baseline + sigma.extraordinary + sigma.ordinary
242+
+ sigma.acceptable + sigma.unacceptable;
243+
244+
if sigma.baseline == total {
245+
println!(" BASELINE RUN — no prior measurements. Record the ns values above.");
246+
println!(" Copy them into the BASELINES struct and recompile for CI runs.");
247+
} else {
248+
println!(" Baseline: {}", sigma.baseline);
249+
println!(" Extraordinary: {}", sigma.extraordinary);
250+
println!(" Ordinary: {}", sigma.ordinary);
251+
println!(" Acceptable: {} (soft fail)", sigma.acceptable);
252+
println!(" Unacceptable: {} (HARD FAIL)", sigma.unacceptable);
253+
println!();
254+
if sigma.unacceptable > 0 {
255+
println!(" RESULT: FAIL — {} hard regression(s)", sigma.unacceptable);
256+
std::process::exit(1);
257+
} else if sigma.acceptable > 0 {
258+
println!(" RESULT: WARN — {} soft regression(s), no hard fails", sigma.acceptable);
259+
} else {
260+
println!(" RESULT: PASS");
261+
}
262+
}
263+
264+
println!("\n=== Done ===");
265+
}
266+
267+
// ── Lexer proxy ────────────────────────────────────────────────────────────────
268+
// Lightweight character-level token scanner. Replace with the real
269+
// tangle::lexer::Lexer when compiling as part of the crate.
270+
fn scan_tokens(src: &str) -> Vec<(usize, usize)> {
271+
let bytes = src.as_bytes();
272+
let mut tokens = Vec::new();
273+
let mut i = 0;
274+
while i < bytes.len() {
275+
match bytes[i] {
276+
b' ' | b'\t' | b'\n' | b'\r' => { i += 1; }
277+
b'#' => { while i < bytes.len() && bytes[i] != b'\n' { i += 1; } }
278+
b'"' => {
279+
let start = i;
280+
i += 1;
281+
while i < bytes.len() && bytes[i] != b'"' { i += 1; }
282+
i += 1;
283+
tokens.push((start, i));
284+
}
285+
c if c.is_ascii_alphabetic() || c == b'_' => {
286+
let start = i;
287+
while i < bytes.len() && (bytes[i].is_ascii_alphanumeric() || bytes[i] == b'_') {
288+
i += 1;
289+
}
290+
tokens.push((start, i));
291+
}
292+
c if c.is_ascii_digit() => {
293+
let start = i;
294+
while i < bytes.len() && bytes[i].is_ascii_digit() { i += 1; }
295+
tokens.push((start, i));
296+
}
297+
_ => { tokens.push((i, i + 1)); i += 1; }
298+
}
299+
}
300+
tokens
301+
}

0 commit comments

Comments
 (0)