Skip to content

Commit 23b47f4

Browse files
aepfliclaude
aepfli
and
claude
authored
perf(java): add concurrency benchmarks C1-C6 (#94)
## Summary - Adds JMH concurrency benchmarks covering 1/4/8 threads, targeting, mixed workload, and read/write contention - Uses `@Group`/`@GroupThreads` for asymmetric C6 (3 readers + 1 writer) - Reveals pre-eval cache scales linearly (44M→54M ops/s), while synchronized WASM is the bottleneck (82K ops/s at 4 threads) ## Benchmarks Added | ID | Threads | Workload | Throughput | |----|---------|----------|-----------| | C1 | 1 | Simple flag (baseline) | 44M ops/s | | C2 | 4 | Simple flag | 47M ops/s | | C3 | 8 | Simple flag | 54M ops/s | | C4 | 4 | Targeting flag | 82K ops/s | | C5 | 4 | Mixed (static+targeting+disabled) | 268K ops/s | | C6 | 3+1 | Read/write contention | 139K read, 343 write ops/s | ## Run ```bash cd java && ./mvnw clean package java -jar target/benchmarks.jar ConcurrencyBenchmark ``` ## Test plan - [x] `./mvnw clean package -DskipTests` compiles successfully - [x] `./mvnw test` — all 35 existing tests pass - [x] Smoke test: all 6 benchmarks produce valid results Closes #90 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 033f287 commit 23b47f4

1 file changed

Lines changed: 383 additions & 0 deletions

File tree

‎java/src/main/java/dev/openfeature/flagd/evaluator/ConcurrencyBenchmark.java‎

Lines changed: 383 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,383 @@
1+
package dev.openfeature.flagd.evaluator;
2+
3+
import dev.openfeature.sdk.EvaluationContext;
4+
import dev.openfeature.sdk.MutableContext;
5+
import org.openjdk.jmh.annotations.*;
6+
import org.openjdk.jmh.infra.Blackhole;
7+
8+
import java.util.concurrent.TimeUnit;
9+
10+
/**
11+
* JMH concurrency benchmarks C1-C6 for FlagEvaluator.
12+
*
13+
* <p>Measures the impact of {@code synchronized} methods on throughput as thread
14+
* count increases. All threads share a single {@link FlagEvaluator} instance,
15+
* matching the production deployment pattern.
16+
*
17+
* <p><b>Benchmark matrix:</b>
18+
* <ul>
19+
* <li>C1: Single thread, simple flag (baseline)</li>
20+
* <li>C2: 4 threads, simple flag</li>
21+
* <li>C3: 8 threads, simple flag</li>
22+
* <li>C4: 4 threads, targeting flag with context</li>
23+
* <li>C5: 4 threads, mixed workload (static, targeting, disabled flags)</li>
24+
* <li>C6: 4 threads read/write contention (3 readers, 1 writer)</li>
25+
* </ul>
26+
*
27+
* <p><b>Running the benchmarks:</b>
28+
* <pre>
29+
* ./mvnw clean package
30+
* java -jar target/benchmarks.jar ConcurrencyBenchmark
31+
*
32+
* # Run a specific scenario:
33+
* java -jar target/benchmarks.jar "ConcurrencyBenchmark.C1_.*"
34+
*
35+
* # Run C6 group benchmark:
36+
* java -jar target/benchmarks.jar "ConcurrencyBenchmark.C6_.*"
37+
* </pre>
38+
*/
39+
@BenchmarkMode(Mode.Throughput)
40+
@OutputTimeUnit(TimeUnit.SECONDS)
41+
@Fork(1)
42+
@Warmup(iterations = 3, time = 2)
43+
@Measurement(iterations = 5, time = 3)
44+
public class ConcurrencyBenchmark {
45+
46+
// Simple flag: static boolean, no targeting
47+
private static final String SIMPLE_FLAG_CONFIG = "{" +
48+
"\"flags\": {" +
49+
" \"simpleFlag\": {" +
50+
" \"state\": \"ENABLED\"," +
51+
" \"variants\": {\"on\": true, \"off\": false}," +
52+
" \"defaultVariant\": \"on\"" +
53+
" }" +
54+
"}" +
55+
"}";
56+
57+
// Targeting flag: evaluates context to resolve variant
58+
private static final String TARGETING_FLAG_CONFIG = "{" +
59+
"\"flags\": {" +
60+
" \"targetingFlag\": {" +
61+
" \"state\": \"ENABLED\"," +
62+
" \"variants\": {\"blue\": \"blue\", \"red\": \"red\"}," +
63+
" \"defaultVariant\": \"red\"," +
64+
" \"targeting\": {" +
65+
" \"if\": [{\"==\": [{\"var\": \"color\"}, \"blue\"]}, \"blue\", \"red\"]" +
66+
" }" +
67+
" }" +
68+
"}" +
69+
"}";
70+
71+
// Mixed config: static + targeting + disabled flags
72+
private static final String MIXED_FLAG_CONFIG = "{" +
73+
"\"flags\": {" +
74+
" \"staticFlag\": {" +
75+
" \"state\": \"ENABLED\"," +
76+
" \"variants\": {\"on\": true, \"off\": false}," +
77+
" \"defaultVariant\": \"on\"" +
78+
" }," +
79+
" \"targetingFlag\": {" +
80+
" \"state\": \"ENABLED\"," +
81+
" \"variants\": {\"blue\": \"blue\", \"red\": \"red\"}," +
82+
" \"defaultVariant\": \"red\"," +
83+
" \"targeting\": {" +
84+
" \"if\": [{\"==\": [{\"var\": \"color\"}, \"blue\"]}, \"blue\", \"red\"]" +
85+
" }" +
86+
" }," +
87+
" \"disabledFlag\": {" +
88+
" \"state\": \"DISABLED\"," +
89+
" \"variants\": {\"on\": true, \"off\": false}," +
90+
" \"defaultVariant\": \"off\"" +
91+
" }" +
92+
"}" +
93+
"}";
94+
95+
// Alternate config for C6 write contention (different default variant)
96+
private static final String MIXED_FLAG_CONFIG_ALT = "{" +
97+
"\"flags\": {" +
98+
" \"staticFlag\": {" +
99+
" \"state\": \"ENABLED\"," +
100+
" \"variants\": {\"on\": true, \"off\": false}," +
101+
" \"defaultVariant\": \"off\"" +
102+
" }," +
103+
" \"targetingFlag\": {" +
104+
" \"state\": \"ENABLED\"," +
105+
" \"variants\": {\"blue\": \"blue\", \"red\": \"red\"}," +
106+
" \"defaultVariant\": \"blue\"," +
107+
" \"targeting\": {" +
108+
" \"if\": [{\"==\": [{\"var\": \"color\"}, \"blue\"]}, \"blue\", \"red\"]" +
109+
" }" +
110+
" }," +
111+
" \"disabledFlag\": {" +
112+
" \"state\": \"DISABLED\"," +
113+
" \"variants\": {\"on\": true, \"off\": false}," +
114+
" \"defaultVariant\": \"off\"" +
115+
" }" +
116+
"}" +
117+
"}";
118+
119+
// Flag keys for C5 mixed workload rotation
120+
private static final String[] MIXED_FLAG_KEYS = {"staticFlag", "targetingFlag", "disabledFlag"};
121+
122+
// ========================================================================
123+
// Shared State: single FlagEvaluator shared across all threads
124+
// ========================================================================
125+
126+
@State(Scope.Benchmark)
127+
public static class SimpleState {
128+
FlagEvaluator evaluator;
129+
130+
@Setup(Level.Trial)
131+
public void setup() {
132+
try {
133+
evaluator = new FlagEvaluator(FlagEvaluator.ValidationMode.PERMISSIVE);
134+
UpdateStateResult result = evaluator.updateState(SIMPLE_FLAG_CONFIG);
135+
if (!result.isSuccess()) {
136+
throw new RuntimeException("Failed to load simple flag config: " + result.getError());
137+
}
138+
} catch (Exception e) {
139+
throw new RuntimeException("Failed to setup SimpleState", e);
140+
}
141+
}
142+
143+
@TearDown(Level.Trial)
144+
public void tearDown() {
145+
if (evaluator != null) {
146+
evaluator.close();
147+
}
148+
}
149+
}
150+
151+
@State(Scope.Benchmark)
152+
public static class TargetingState {
153+
FlagEvaluator evaluator;
154+
155+
@Setup(Level.Trial)
156+
public void setup() {
157+
try {
158+
evaluator = new FlagEvaluator(FlagEvaluator.ValidationMode.PERMISSIVE);
159+
UpdateStateResult result = evaluator.updateState(TARGETING_FLAG_CONFIG);
160+
if (!result.isSuccess()) {
161+
throw new RuntimeException("Failed to load targeting flag config: " + result.getError());
162+
}
163+
} catch (Exception e) {
164+
throw new RuntimeException("Failed to setup TargetingState", e);
165+
}
166+
}
167+
168+
@TearDown(Level.Trial)
169+
public void tearDown() {
170+
if (evaluator != null) {
171+
evaluator.close();
172+
}
173+
}
174+
}
175+
176+
@State(Scope.Benchmark)
177+
public static class MixedState {
178+
FlagEvaluator evaluator;
179+
180+
@Setup(Level.Trial)
181+
public void setup() {
182+
try {
183+
evaluator = new FlagEvaluator(FlagEvaluator.ValidationMode.PERMISSIVE);
184+
UpdateStateResult result = evaluator.updateState(MIXED_FLAG_CONFIG);
185+
if (!result.isSuccess()) {
186+
throw new RuntimeException("Failed to load mixed flag config: " + result.getError());
187+
}
188+
} catch (Exception e) {
189+
throw new RuntimeException("Failed to setup MixedState", e);
190+
}
191+
}
192+
193+
@TearDown(Level.Trial)
194+
public void tearDown() {
195+
if (evaluator != null) {
196+
evaluator.close();
197+
}
198+
}
199+
}
200+
201+
@State(Scope.Benchmark)
202+
public static class ReadWriteState {
203+
FlagEvaluator evaluator;
204+
205+
@Setup(Level.Trial)
206+
public void setup() {
207+
try {
208+
evaluator = new FlagEvaluator(FlagEvaluator.ValidationMode.PERMISSIVE);
209+
UpdateStateResult result = evaluator.updateState(MIXED_FLAG_CONFIG);
210+
if (!result.isSuccess()) {
211+
throw new RuntimeException("Failed to load config for read/write state: " + result.getError());
212+
}
213+
} catch (Exception e) {
214+
throw new RuntimeException("Failed to setup ReadWriteState", e);
215+
}
216+
}
217+
218+
@TearDown(Level.Trial)
219+
public void tearDown() {
220+
if (evaluator != null) {
221+
evaluator.close();
222+
}
223+
}
224+
}
225+
226+
// ========================================================================
227+
// Thread-local State: per-thread context and rotation counter
228+
// ========================================================================
229+
230+
@State(Scope.Thread)
231+
public static class ThreadContext {
232+
EvaluationContext targetingContext;
233+
String simpleContextJson;
234+
int counter;
235+
236+
@Setup(Level.Trial)
237+
public void setup() {
238+
targetingContext = new MutableContext()
239+
.add("color", "blue")
240+
.add("targetingKey", "user-123");
241+
242+
simpleContextJson = "{\"targetingKey\": \"user-123\"}";
243+
counter = 0;
244+
}
245+
}
246+
247+
// ========================================================================
248+
// C1: Single thread, simple flag (baseline)
249+
// ========================================================================
250+
251+
@Benchmark
252+
@Threads(1)
253+
public void C1_singleThread_simpleFlag(SimpleState state, ThreadContext ctx, Blackhole bh) {
254+
try {
255+
EvaluationResult<Boolean> result = state.evaluator.evaluateFlag(
256+
Boolean.class, "simpleFlag", ctx.simpleContextJson);
257+
bh.consume(result.getValue());
258+
bh.consume(result.getVariant());
259+
} catch (Exception e) {
260+
throw new RuntimeException("C1 benchmark failed", e);
261+
}
262+
}
263+
264+
// ========================================================================
265+
// C2: 4 threads, simple flag
266+
// ========================================================================
267+
268+
@Benchmark
269+
@Threads(4)
270+
public void C2_4threads_simpleFlag(SimpleState state, ThreadContext ctx, Blackhole bh) {
271+
try {
272+
EvaluationResult<Boolean> result = state.evaluator.evaluateFlag(
273+
Boolean.class, "simpleFlag", ctx.simpleContextJson);
274+
bh.consume(result.getValue());
275+
bh.consume(result.getVariant());
276+
} catch (Exception e) {
277+
throw new RuntimeException("C2 benchmark failed", e);
278+
}
279+
}
280+
281+
// ========================================================================
282+
// C3: 8 threads, simple flag
283+
// ========================================================================
284+
285+
@Benchmark
286+
@Threads(8)
287+
public void C3_8threads_simpleFlag(SimpleState state, ThreadContext ctx, Blackhole bh) {
288+
try {
289+
EvaluationResult<Boolean> result = state.evaluator.evaluateFlag(
290+
Boolean.class, "simpleFlag", ctx.simpleContextJson);
291+
bh.consume(result.getValue());
292+
bh.consume(result.getVariant());
293+
} catch (Exception e) {
294+
throw new RuntimeException("C3 benchmark failed", e);
295+
}
296+
}
297+
298+
// ========================================================================
299+
// C4: 4 threads, targeting flag with context
300+
// ========================================================================
301+
302+
@Benchmark
303+
@Threads(4)
304+
public void C4_4threads_targetingFlag(TargetingState state, ThreadContext ctx, Blackhole bh) {
305+
try {
306+
EvaluationResult<String> result = state.evaluator.evaluateFlag(
307+
String.class, "targetingFlag", ctx.targetingContext);
308+
bh.consume(result.getValue());
309+
bh.consume(result.getVariant());
310+
} catch (Exception e) {
311+
throw new RuntimeException("C4 benchmark failed", e);
312+
}
313+
}
314+
315+
// ========================================================================
316+
// C5: 4 threads, mixed workload (rotate through static, targeting, disabled)
317+
// ========================================================================
318+
319+
@Benchmark
320+
@Threads(4)
321+
public void C5_4threads_mixedWorkload(MixedState state, ThreadContext ctx, Blackhole bh) {
322+
try {
323+
String flagKey = MIXED_FLAG_KEYS[ctx.counter % MIXED_FLAG_KEYS.length];
324+
ctx.counter++;
325+
326+
if ("targetingFlag".equals(flagKey)) {
327+
EvaluationResult<String> result = state.evaluator.evaluateFlag(
328+
String.class, flagKey, ctx.targetingContext);
329+
bh.consume(result.getValue());
330+
bh.consume(result.getVariant());
331+
} else {
332+
EvaluationResult<Boolean> result = state.evaluator.evaluateFlag(
333+
Boolean.class, flagKey, ctx.simpleContextJson);
334+
bh.consume(result.getValue());
335+
bh.consume(result.getVariant());
336+
}
337+
} catch (Exception e) {
338+
throw new RuntimeException("C5 benchmark failed", e);
339+
}
340+
}
341+
342+
// ========================================================================
343+
// C6: 4 threads read/write contention (3 readers, 1 writer)
344+
// Uses @Group/@GroupThreads for asymmetric workload.
345+
// ========================================================================
346+
347+
@Benchmark
348+
@Group("C6_readWriteContention")
349+
@GroupThreads(3)
350+
public void C6_read(ReadWriteState state, ThreadContext ctx, Blackhole bh) {
351+
try {
352+
EvaluationResult<Boolean> result = state.evaluator.evaluateFlag(
353+
Boolean.class, "staticFlag", ctx.simpleContextJson);
354+
bh.consume(result.getValue());
355+
bh.consume(result.getVariant());
356+
} catch (Exception e) {
357+
throw new RuntimeException("C6 read benchmark failed", e);
358+
}
359+
}
360+
361+
@Benchmark
362+
@Group("C6_readWriteContention")
363+
@GroupThreads(1)
364+
public void C6_write(ReadWriteState state, ThreadContext ctx, Blackhole bh) {
365+
try {
366+
// Alternate between two configs to force actual state changes
367+
String config = (ctx.counter % 2 == 0)
368+
? MIXED_FLAG_CONFIG : MIXED_FLAG_CONFIG_ALT;
369+
ctx.counter++;
370+
UpdateStateResult result = state.evaluator.updateState(config);
371+
bh.consume(result.isSuccess());
372+
} catch (Exception e) {
373+
throw new RuntimeException("C6 write benchmark failed", e);
374+
}
375+
}
376+
377+
/**
378+
* Main method to run benchmarks standalone.
379+
*/
380+
public static void main(String[] args) throws Exception {
381+
org.openjdk.jmh.Main.main(args);
382+
}
383+
}

0 commit comments

Comments
 (0)