Skip to content

Commit 044f4c3

Browse files
aepfliclaude
andauthored
perf(rust): add scale benchmarks S6-S8, S10-S11 for large flag stores (#93)
## Summary - Adds criterion benchmarks for large flag configurations (1K–100K flags) - Confirms O(1) evaluation lookup at scale — no degradation with 10K flags in store - Flag generation uses production-like distribution: 70% static, 25% targeting, 5% disabled ## Benchmarks Added | ID | Scenario | Result | |----|----------|--------| | S6 | `update_state` with 1,000 flags | ~9.3 ms | | S7 | `update_state` with 10,000 flags | (longer run) | | S8 | `update_state` with 100,000 flags | (longer run) | | S10 | Evaluate static flag from 10K store | ~101 ns | | S11 | Evaluate targeting flag from 10K store | ~665 ns | S9 (1M flags) skipped as it may be too slow for CI. ## Run ```bash cargo bench -- scale ``` ## Test plan - [x] `cargo bench -- S6` runs and produces results - [x] S10/S11 confirm O(1) lookup (comparable to baseline evaluation benchmarks) - [x] `cargo test --lib -- --test-threads=1` — all existing tests pass Closes #86 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 23b47f4 commit 044f4c3

3 files changed

Lines changed: 301 additions & 2 deletions

File tree

Cargo.lock

Lines changed: 167 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ harness = false
6060
name = "comparison"
6161
harness = false
6262

63+
[[bench]]
64+
name = "scale"
65+
harness = false
66+
6367
[profile.release]
6468
# Optimize for speed instead of size for better runtime performance
6569
opt-level = 2

benches/scale.rs

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
//! Scale benchmarks for large flag stores (S6-S11).
2+
//!
3+
//! Tests update_state performance at 1K, 10K, and 100K flags, and verifies
4+
//! O(1) evaluation lookup from a 10K-flag store for both static and targeting flags.
5+
//! Flag distributions approximate production workloads: 70% static, 25% targeting, 5% disabled.
6+
7+
use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
8+
use flagd_evaluator::{FlagEvaluator, ValidationMode};
9+
use serde_json::json;
10+
use std::time::Duration;
11+
12+
/// Generates a flag configuration JSON string with the specified number of flags.
13+
///
14+
/// Distribution:
15+
/// - 70% static (ENABLED, no targeting)
16+
/// - 25% simple targeting (single `==` condition)
17+
/// - 5% disabled
18+
///
19+
/// Flag names follow the pattern `flag_0001`, `flag_0002`, etc.
20+
fn generate_scale_config(num_flags: usize) -> String {
21+
let mut buf = String::with_capacity(num_flags * 200);
22+
buf.push_str(r#"{"flags":{"#);
23+
24+
for i in 0..num_flags {
25+
if i > 0 {
26+
buf.push(',');
27+
}
28+
29+
let name = format!("flag_{:04}", i);
30+
let category = i % 20; // deterministic distribution
31+
32+
if category < 14 {
33+
// 70% static flags (0-13 out of 0-19)
34+
buf.push_str(&format!(
35+
r#""{name}":{{"state":"ENABLED","variants":{{"on":true,"off":false}},"defaultVariant":"on"}}"#
36+
));
37+
} else if category < 19 {
38+
// 25% targeting flags (14-18 out of 0-19)
39+
buf.push_str(&format!(
40+
r#""{name}":{{"state":"ENABLED","variants":{{"on":true,"off":false}},"defaultVariant":"off","targeting":{{"if":[{{"==":[{{"var":"color"}},"blue"]}},"on","off"]}}}}"#
41+
));
42+
} else {
43+
// 5% disabled flags (19 out of 0-19)
44+
buf.push_str(&format!(
45+
r#""{name}":{{"state":"DISABLED","variants":{{"on":true,"off":false}},"defaultVariant":"on"}}"#
46+
));
47+
}
48+
}
49+
50+
buf.push_str("}}");
51+
buf
52+
}
53+
54+
// ---------------------------------------------------------------------------
55+
// S6-S8: update_state at scale
56+
// ---------------------------------------------------------------------------
57+
58+
fn bench_update_state_scale(c: &mut Criterion) {
59+
let mut group = c.benchmark_group("scale_update_state");
60+
61+
for &(size, label, sample_size) in &[
62+
(1_000, "S6_1K", 50),
63+
(10_000, "S7_10K", 10),
64+
(100_000, "S8_100K", 10),
65+
] {
66+
let config = generate_scale_config(size);
67+
68+
group.sample_size(sample_size);
69+
// Give S8 more time to complete
70+
if size >= 100_000 {
71+
group.measurement_time(Duration::from_secs(30));
72+
} else if size >= 10_000 {
73+
group.measurement_time(Duration::from_secs(15));
74+
}
75+
76+
group.bench_with_input(BenchmarkId::new("fresh", label), &config, |b, config| {
77+
b.iter_batched(
78+
|| FlagEvaluator::new(ValidationMode::Permissive),
79+
|mut evaluator| {
80+
evaluator.update_state(black_box(config)).unwrap();
81+
},
82+
criterion::BatchSize::PerIteration,
83+
)
84+
});
85+
}
86+
87+
group.finish();
88+
}
89+
90+
// ---------------------------------------------------------------------------
91+
// S10-S11: Evaluate from a large (10K) flag store
92+
// ---------------------------------------------------------------------------
93+
94+
/// S10: Evaluate a static flag from a 10K-flag store.
95+
/// Verifies that flag lookup is O(1) regardless of store size.
96+
fn bench_evaluate_static_from_10k(c: &mut Criterion) {
97+
let config = generate_scale_config(10_000);
98+
let mut evaluator = FlagEvaluator::new(ValidationMode::Permissive);
99+
evaluator.update_state(&config).unwrap();
100+
101+
// flag_0000 is a static flag (index 0, 0 % 20 == 0, which is < 14 -> static)
102+
let context = json!({});
103+
104+
c.bench_function("S10_evaluate_static_from_10K_store", |b| {
105+
b.iter(|| evaluator.evaluate_flag(black_box("flag_0000"), black_box(&context)))
106+
});
107+
}
108+
109+
/// S11: Evaluate a targeting flag from a 10K-flag store.
110+
/// Verifies that targeting evaluation is O(1) regardless of store size.
111+
fn bench_evaluate_targeting_from_10k(c: &mut Criterion) {
112+
let config = generate_scale_config(10_000);
113+
let mut evaluator = FlagEvaluator::new(ValidationMode::Permissive);
114+
evaluator.update_state(&config).unwrap();
115+
116+
// flag_0014 is a targeting flag (index 14, 14 % 20 == 14, which is >= 14 and < 19 -> targeting)
117+
let context = json!({"color": "blue", "targetingKey": "user-123"});
118+
119+
c.bench_function("S11_evaluate_targeting_from_10K_store", |b| {
120+
b.iter(|| evaluator.evaluate_flag(black_box("flag_0014"), black_box(&context)))
121+
});
122+
}
123+
124+
criterion_group!(
125+
benches,
126+
bench_update_state_scale,
127+
bench_evaluate_static_from_10k,
128+
bench_evaluate_targeting_from_10k,
129+
);
130+
criterion_main!(benches);

0 commit comments

Comments
 (0)