|
| 1 | +//! Scale benchmarks for large flag stores (S6-S11). |
| 2 | +//! |
| 3 | +//! Tests update_state performance at 1K, 10K, and 100K flags, and verifies |
| 4 | +//! O(1) evaluation lookup from a 10K-flag store for both static and targeting flags. |
| 5 | +//! Flag distributions approximate production workloads: 70% static, 25% targeting, 5% disabled. |
| 6 | +
|
| 7 | +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; |
| 8 | +use flagd_evaluator::{FlagEvaluator, ValidationMode}; |
| 9 | +use serde_json::json; |
| 10 | +use std::time::Duration; |
| 11 | + |
| 12 | +/// Generates a flag configuration JSON string with the specified number of flags. |
| 13 | +/// |
| 14 | +/// Distribution: |
| 15 | +/// - 70% static (ENABLED, no targeting) |
| 16 | +/// - 25% simple targeting (single `==` condition) |
| 17 | +/// - 5% disabled |
| 18 | +/// |
| 19 | +/// Flag names follow the pattern `flag_0001`, `flag_0002`, etc. |
| 20 | +fn generate_scale_config(num_flags: usize) -> String { |
| 21 | + let mut buf = String::with_capacity(num_flags * 200); |
| 22 | + buf.push_str(r#"{"flags":{"#); |
| 23 | + |
| 24 | + for i in 0..num_flags { |
| 25 | + if i > 0 { |
| 26 | + buf.push(','); |
| 27 | + } |
| 28 | + |
| 29 | + let name = format!("flag_{:04}", i); |
| 30 | + let category = i % 20; // deterministic distribution |
| 31 | + |
| 32 | + if category < 14 { |
| 33 | + // 70% static flags (0-13 out of 0-19) |
| 34 | + buf.push_str(&format!( |
| 35 | + r#""{name}":{{"state":"ENABLED","variants":{{"on":true,"off":false}},"defaultVariant":"on"}}"# |
| 36 | + )); |
| 37 | + } else if category < 19 { |
| 38 | + // 25% targeting flags (14-18 out of 0-19) |
| 39 | + buf.push_str(&format!( |
| 40 | + r#""{name}":{{"state":"ENABLED","variants":{{"on":true,"off":false}},"defaultVariant":"off","targeting":{{"if":[{{"==":[{{"var":"color"}},"blue"]}},"on","off"]}}}}"# |
| 41 | + )); |
| 42 | + } else { |
| 43 | + // 5% disabled flags (19 out of 0-19) |
| 44 | + buf.push_str(&format!( |
| 45 | + r#""{name}":{{"state":"DISABLED","variants":{{"on":true,"off":false}},"defaultVariant":"on"}}"# |
| 46 | + )); |
| 47 | + } |
| 48 | + } |
| 49 | + |
| 50 | + buf.push_str("}}"); |
| 51 | + buf |
| 52 | +} |
| 53 | + |
| 54 | +// --------------------------------------------------------------------------- |
| 55 | +// S6-S8: update_state at scale |
| 56 | +// --------------------------------------------------------------------------- |
| 57 | + |
| 58 | +fn bench_update_state_scale(c: &mut Criterion) { |
| 59 | + let mut group = c.benchmark_group("scale_update_state"); |
| 60 | + |
| 61 | + for &(size, label, sample_size) in &[ |
| 62 | + (1_000, "S6_1K", 50), |
| 63 | + (10_000, "S7_10K", 10), |
| 64 | + (100_000, "S8_100K", 10), |
| 65 | + ] { |
| 66 | + let config = generate_scale_config(size); |
| 67 | + |
| 68 | + group.sample_size(sample_size); |
| 69 | + // Give S8 more time to complete |
| 70 | + if size >= 100_000 { |
| 71 | + group.measurement_time(Duration::from_secs(30)); |
| 72 | + } else if size >= 10_000 { |
| 73 | + group.measurement_time(Duration::from_secs(15)); |
| 74 | + } |
| 75 | + |
| 76 | + group.bench_with_input(BenchmarkId::new("fresh", label), &config, |b, config| { |
| 77 | + b.iter_batched( |
| 78 | + || FlagEvaluator::new(ValidationMode::Permissive), |
| 79 | + |mut evaluator| { |
| 80 | + evaluator.update_state(black_box(config)).unwrap(); |
| 81 | + }, |
| 82 | + criterion::BatchSize::PerIteration, |
| 83 | + ) |
| 84 | + }); |
| 85 | + } |
| 86 | + |
| 87 | + group.finish(); |
| 88 | +} |
| 89 | + |
| 90 | +// --------------------------------------------------------------------------- |
| 91 | +// S10-S11: Evaluate from a large (10K) flag store |
| 92 | +// --------------------------------------------------------------------------- |
| 93 | + |
| 94 | +/// S10: Evaluate a static flag from a 10K-flag store. |
| 95 | +/// Verifies that flag lookup is O(1) regardless of store size. |
| 96 | +fn bench_evaluate_static_from_10k(c: &mut Criterion) { |
| 97 | + let config = generate_scale_config(10_000); |
| 98 | + let mut evaluator = FlagEvaluator::new(ValidationMode::Permissive); |
| 99 | + evaluator.update_state(&config).unwrap(); |
| 100 | + |
| 101 | + // flag_0000 is a static flag (index 0, 0 % 20 == 0, which is < 14 -> static) |
| 102 | + let context = json!({}); |
| 103 | + |
| 104 | + c.bench_function("S10_evaluate_static_from_10K_store", |b| { |
| 105 | + b.iter(|| evaluator.evaluate_flag(black_box("flag_0000"), black_box(&context))) |
| 106 | + }); |
| 107 | +} |
| 108 | + |
| 109 | +/// S11: Evaluate a targeting flag from a 10K-flag store. |
| 110 | +/// Verifies that targeting evaluation is O(1) regardless of store size. |
| 111 | +fn bench_evaluate_targeting_from_10k(c: &mut Criterion) { |
| 112 | + let config = generate_scale_config(10_000); |
| 113 | + let mut evaluator = FlagEvaluator::new(ValidationMode::Permissive); |
| 114 | + evaluator.update_state(&config).unwrap(); |
| 115 | + |
| 116 | + // flag_0014 is a targeting flag (index 14, 14 % 20 == 14, which is >= 14 and < 19 -> targeting) |
| 117 | + let context = json!({"color": "blue", "targetingKey": "user-123"}); |
| 118 | + |
| 119 | + c.bench_function("S11_evaluate_targeting_from_10K_store", |b| { |
| 120 | + b.iter(|| evaluator.evaluate_flag(black_box("flag_0014"), black_box(&context))) |
| 121 | + }); |
| 122 | +} |
| 123 | + |
| 124 | +criterion_group!( |
| 125 | + benches, |
| 126 | + bench_update_state_scale, |
| 127 | + bench_evaluate_static_from_10k, |
| 128 | + bench_evaluate_targeting_from_10k, |
| 129 | +); |
| 130 | +criterion_main!(benches); |
0 commit comments