Skip to content

Commit 73b19dd

Browse files
dougqhdougqh
committed
Adding benchmarks to illustrate recommended approach to common tracer situations
1 parent f453b4e commit 73b19dd

5 files changed

Lines changed: 805 additions & 1 deletion

File tree

‎internal-api/src/jmh/java/datadog/trace/util/CaseInsensitiveMapBenchmark.java‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
package datadog.trace.util;
2+
3+
import java.util.HashMap;
4+
import java.util.TreeMap;
5+
import java.util.concurrent.ThreadLocalRandom;
6+
import java.util.function.Supplier;
7+
8+
import org.openjdk.jmh.annotations.Benchmark;
9+
import org.openjdk.jmh.annotations.Fork;
10+
import org.openjdk.jmh.annotations.Measurement;
11+
import org.openjdk.jmh.annotations.Threads;
12+
import org.openjdk.jmh.annotations.Warmup;
13+
import org.openjdk.jmh.infra.Blackhole;
14+
15+
/**
16+
* <ul>Benchmark to illustrate the trade-offs around case-insensitive Map look-ups - using either...
17+
* <li>(RECOMMENDED) TreeMap with Comparator of String::compareToIgnoreCase
18+
* <li>HashMap with look-ups using String::to<X>Case
19+
* </ul>
20+
*
21+
* <p>For case-insensitive lookups, TreeMap map creation is consistently faster because it
22+
* avoids String::to<X>Case calls.
23+
*
24+
* <p>Despite calls to String::to<X>Case, HashMap lookups are faster in single threaded
25+
* microbenchmark by 50% but are worse when frequently called in a multi-threaded system.
26+
*
27+
* <p>With many threads, the extra allocation from calling String::to<X>Case leads to frequent GCs
28+
* which has adverse impacts on the whole system.
29+
*
30+
* <code>
31+
* MacBook M1 with 1 thread (Java 21)
32+
*
33+
* Benchmark Mode Cnt Score Error Units
34+
* CaseInsensitiveMapBenchmark.create_hashMap thrpt 6 994213.041 ± 15718.903 ops/s
35+
* CaseInsensitiveMapBenchmark.create_treeMap thrpt 6 1522900.015 ± 21646.688 ops/s
36+
*
37+
* CaseInsensitiveMapBenchmark.get_hashMap thrpt 6 69149862.293 ± 9168648.566 ops/s
38+
* CaseInsensitiveMapBenchmark.get_treeMap thrpt 6 42796699.230 ± 9029447.805 ops/s
39+
* </code>
40+
*
41+
* <code>
42+
* MacBook M1 with 8 threads (Java 21)
43+
*
44+
* Benchmark Mode Cnt Score Error Units
45+
* CaseInsensitiveMapBenchmark.create_hashMap thrpt 6 6641003.483 ± 543210.409 ops/s
46+
* CaseInsensitiveMapBenchmark.create_treeMap thrpt 6 10030191.764 ± 1308865.113 ops/s
47+
*
48+
* CaseInsensitiveMapBenchmark.get_hashMap thrpt 6 38748031.837 ± 9012072.804 ops/s
49+
* CaseInsensitiveMapBenchmark.get_treeMap thrpt 6 173495470.789 ± 27824904.999 ops/s
50+
* </code>
51+
*/
52+
@Fork(2)
53+
@Warmup(iterations=2)
54+
@Measurement(iterations=3)
55+
@Threads(8)
56+
public class CaseInsensitiveMapBenchmark {
57+
static final String[] PREFIXES = {
58+
"foo",
59+
"bar",
60+
"baz",
61+
"quux"
62+
};
63+
64+
static final int NUM_SUFFIXES = 4;
65+
66+
static <T> T init(Supplier<T> supplier) {
67+
return supplier.get();
68+
}
69+
70+
static final String[] UPPER_PREFIXES = init(() -> {
71+
String[] upperPrefixes = new String[PREFIXES.length];
72+
for ( int i = 0; i < PREFIXES.length; ++i ) {
73+
upperPrefixes[i] = PREFIXES[i].toUpperCase();
74+
}
75+
return upperPrefixes;
76+
});
77+
78+
static final String[] LOOKUP_KEYS = init(() -> {
79+
ThreadLocalRandom curRandom = ThreadLocalRandom.current();
80+
81+
String[] keys = new String[32];
82+
for ( int i = 0; i < keys.length; ++i ) {
83+
int prefixIndex = curRandom.nextInt(PREFIXES.length);
84+
boolean toUpper = curRandom.nextBoolean();
85+
int suffixIndex = curRandom.nextInt(NUM_SUFFIXES + 1);
86+
87+
String key = PREFIXES[prefixIndex] + "-" + suffixIndex;
88+
keys[i] = toUpper ? key.toUpperCase() : key.toLowerCase();
89+
}
90+
return keys;
91+
});
92+
93+
static int sharedLookupIndex = 0;
94+
95+
static String nextLookupKey() {
96+
int localIndex = ++sharedLookupIndex;
97+
if ( localIndex >= LOOKUP_KEYS.length ) {
98+
sharedLookupIndex = localIndex = 0;
99+
}
100+
return LOOKUP_KEYS[localIndex];
101+
}
102+
103+
@Benchmark
104+
public void create_baseline(Blackhole blackhole) {
105+
for ( int suffix = 0; suffix < NUM_SUFFIXES; ++suffix ) {
106+
for ( String prefix: PREFIXES ) {
107+
blackhole.consume(prefix + "-" + suffix);
108+
blackhole.consume(Integer.valueOf(suffix));
109+
}
110+
}
111+
for ( int suffix = 0; suffix < NUM_SUFFIXES; suffix +=2 ) {
112+
for ( String prefix: UPPER_PREFIXES ) {
113+
blackhole.consume(prefix + "-" + suffix);
114+
blackhole.consume(Integer.valueOf(suffix + 1));
115+
}
116+
}
117+
}
118+
119+
@Benchmark
120+
public void lookup_baseline(Blackhole blackhole) {
121+
blackhole.consume(nextLookupKey());
122+
}
123+
124+
@Benchmark
125+
public HashMap<String, Integer> create_hashMap() {
126+
return _create_hashMap();
127+
}
128+
129+
static HashMap<String, Integer> _create_hashMap() {
130+
HashMap<String, Integer> map = new HashMap<>();
131+
for ( int suffix = 0; suffix < NUM_SUFFIXES; ++suffix ) {
132+
for ( String prefix: PREFIXES ) {
133+
map.put((prefix + "-" + suffix).toLowerCase(), suffix); // arguable, but real caller probably doesn't know the case ahead-of-time
134+
}
135+
}
136+
for ( int suffix = 0; suffix < NUM_SUFFIXES; suffix +=2 ) {
137+
for ( String prefix: UPPER_PREFIXES ) {
138+
map.put((prefix + "-" + suffix).toLowerCase(), suffix + 1);
139+
}
140+
}
141+
return map;
142+
}
143+
144+
static final HashMap<String, Integer> HASH_MAP = _create_hashMap();
145+
146+
@Benchmark
147+
public Integer lookup_hashMap() {
148+
// This benchmark is still "correct" in multi-threaded context,
149+
// Map is populated under the class initialization lock and not changed thereafter
150+
return HASH_MAP.get(nextLookupKey().toLowerCase());
151+
}
152+
153+
@Benchmark
154+
public TreeMap<String, Integer> create_treeMap() {
155+
return _create_treeMap();
156+
}
157+
158+
static TreeMap<String, Integer> _create_treeMap() {
159+
TreeMap<String, Integer> map = new TreeMap<>(String::compareToIgnoreCase);
160+
for ( int suffix = 0; suffix < NUM_SUFFIXES; ++suffix ) {
161+
for ( String prefix: PREFIXES ) {
162+
map.put(prefix + "-" + suffix, suffix);
163+
}
164+
}
165+
for ( int suffix = 0; suffix < NUM_SUFFIXES; suffix +=2 ) {
166+
for ( String prefix: UPPER_PREFIXES ) {
167+
map.put(prefix + "-" + suffix, suffix + 1);
168+
}
169+
}
170+
return map;
171+
}
172+
173+
static final TreeMap<String, Integer> TREE_MAP = _create_treeMap();
174+
175+
@Benchmark
176+
public Integer lookup_treeMap() {
177+
// This benchmark is still "correct" in multi-threaded context,
178+
// Map is populated under the initial class initialization lock and not changed thereafter
179+
return TREE_MAP.get(nextLookupKey());
180+
}
181+
182+
// TODO: Add ConcurrentSkipListMap & synchronized HashMap & TreeMap
183+
}

‎internal-api/src/jmh/java/datadog/trace/util/HashingBenchmark.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,12 @@
99
import org.openjdk.jmh.annotations.Warmup;
1010

1111
/**
12-
* In contrast to java.util.Objects.hash, datadog.util.HashingUtils.hash has overrides for different
12+
* <ul>Benchmark comparing HashingUtils.hash to Objects.hash
13+
* <li>(RECOMMENDED) HashingUtils.hash - avoids var-arg creation
14+
* <li>Object.hash - high allocation overhead from var-ags
15+
* </ul>
16+
*
17+
* <p>In contrast to java.util.Objects.hash, datadog.util.HashingUtils.hash has overrides for different
1318
* parameter counts that allow most callers to avoid calling the var-arg version. This avoids the
1419
* common situation where the JIT's escape analysis is unable to elide the var-arg array allocation.
1520
*

‎internal-api/src/jmh/java/datadog/trace/util/SetBenchmark.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package datadog.trace.util;
2+
3+
import java.util.Arrays;
4+
import java.util.Collections;
5+
import java.util.HashSet;
6+
import java.util.TreeSet;
7+
import java.util.concurrent.ThreadLocalRandom;
8+
import java.util.function.Supplier;
9+
10+
import org.openjdk.jmh.annotations.Benchmark;
11+
import org.openjdk.jmh.annotations.Fork;
12+
import org.openjdk.jmh.annotations.Measurement;
13+
import org.openjdk.jmh.annotations.Threads;
14+
import org.openjdk.jmh.annotations.Warmup;
15+
16+
/**
17+
* <ul>Benchmark showing possible ways to represent and check if a set includes an elememt...
18+
* <li>(RECOMMENDED) HashSet - on par with TreeSet - idiomatic
19+
* <li>(RECOMMENDED) TreeMap - on par with HashSet - better solution if custom comparator is needed (see CaseInsensitiveMapBenchmark)
20+
* <li>array - slower than HashSet
21+
* <li>sortedArray - slowest - slower than array for common case of small arrays
22+
* </ul>
23+
*
24+
* <code>
25+
* MacBook M1 - 8 threads - Java 21
26+
* 1/3 not found rate
27+
*
28+
* Benchmark Mode Cnt Score Error Units
29+
* SetBenchmark.contains_array thrpt 6 645561886.327 ± 100781717.494 ops/s
30+
* SetBenchmark.contains_hashSet thrpt 6 1536236680.235 ± 114966961.506 ops/s
31+
* SetBenchmark.contains_sortedArray thrpt 6 571476939.441 ± 21334620.460 ops/s
32+
* SetBenchmark.contains_treeSet thrpt 6 1557663759.411 ± 95343683.124 ops/s
33+
* </code>
34+
*/
35+
@Fork(2)
36+
@Warmup(iterations=2)
37+
@Measurement(iterations=3)
38+
@Threads(8)
39+
public class SetBenchmark {
40+
static final String[] STRINGS = new String[] {
41+
"foo",
42+
"bar",
43+
"baz",
44+
"quux",
45+
"hello",
46+
"world",
47+
"service",
48+
"queryString",
49+
"lorem",
50+
"ipsum",
51+
"dolem",
52+
"sit"
53+
};
54+
55+
static <T> T init(Supplier<T> supplier) {
56+
return supplier.get();
57+
}
58+
59+
static final String[] LOOKUPS = init(() -> {
60+
String[] lookups = Arrays.copyOf(STRINGS, STRINGS.length * 10);
61+
62+
for ( int i = 0; i < STRINGS.length; ++i ) {
63+
lookups[STRINGS.length + i] = new String(STRINGS[i]);
64+
}
65+
66+
// 2 / 3 of the key look-ups miss the set
67+
for ( int i = STRINGS.length * 2; i < lookups.length; ++i ) {
68+
lookups[i] = "dne-" + ThreadLocalRandom.current().nextInt();
69+
}
70+
71+
Collections.shuffle(Arrays.asList(lookups));
72+
return lookups;
73+
});
74+
75+
static int sharedLookupIndex = 0;
76+
77+
static String nextString() {
78+
int localIndex = ++sharedLookupIndex;
79+
if ( localIndex >= LOOKUPS.length ) {
80+
sharedLookupIndex = localIndex = 0;
81+
}
82+
return LOOKUPS[localIndex];
83+
}
84+
85+
static final String[] ARRAY = STRINGS;
86+
87+
@Benchmark
88+
public boolean contains_array() {
89+
String needle = nextString();
90+
for ( String str: ARRAY ) {
91+
if ( needle.equals(str) ) return true;
92+
}
93+
return false;
94+
}
95+
96+
static final String[] SORTED_ARRAY = init(() -> {
97+
String[] sorted = Arrays.copyOf(STRINGS, STRINGS.length);
98+
Arrays.sort(sorted);
99+
return sorted;
100+
});
101+
102+
@Benchmark
103+
public boolean contains_sortedArray() {
104+
return (Arrays.binarySearch(SORTED_ARRAY, nextString()) != -1);
105+
}
106+
107+
static final HashSet<String> HASH_SET = new HashSet<>(Arrays.asList(STRINGS));
108+
109+
@Benchmark
110+
public boolean contains_hashSet() {
111+
return HASH_SET.contains(nextString());
112+
}
113+
114+
static final TreeSet<String> TREE_SET = new TreeSet<>(Arrays.asList(STRINGS));
115+
116+
@Benchmark
117+
public boolean contains_treeSet() {
118+
return HASH_SET.contains(nextString());
119+
}
120+
}

0 commit comments

Comments
 (0)