Skip to content

Commit 163e75e

Browse files
cv4gcv4g
committed
Merge branch 'master'
2 parents 91f6b19 + b3e8cc9 commit 163e75e

4 files changed

Lines changed: 171 additions & 28 deletions

File tree

‎benchmarks/CMakeLists.txt‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -45,15 +45,8 @@ set_target_properties(count_min_sketch_benchmark PROPERTIES
4545
CXX_STANDARD_REQUIRED YES
4646
)
4747

48-
set(COUNT_MIN_SKETCH_BENCHMARK_SOURCES
49-
benchmark_count_min_sketch_serialization.cpp
50-
)
51-
52-
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/benchmark_count_min_sketch.cpp")
53-
list(APPEND COUNT_MIN_SKETCH_BENCHMARK_SOURCES benchmark_count_min_sketch.cpp)
54-
endif()
55-
5648
target_sources(count_min_sketch_benchmark
5749
PRIVATE
58-
${COUNT_MIN_SKETCH_BENCHMARK_SOURCES}
50+
benchmark_count_min_sketch.cpp
51+
benchmark_count_min_sketch_serialization.cpp
5952
)

‎benchmarks/benchmark_count_min_sketch.cpp‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#include <benchmark/benchmark.h>
21+
#include <count_min.hpp>
22+
23+
#include <cstddef>
24+
#include <cstdint>
25+
#include <string>
26+
#include <vector>
27+
28+
namespace
29+
{
30+
31+
using Sketch = datasketches::count_min_sketch<uint64_t>;
32+
33+
// Roughly 99.9% confidence and 0.1% relative error:
34+
// suggest_num_hashes(0.999) == 7 and suggest_num_buckets(0.001) == 2719.
35+
constexpr uint8_t NUM_HASHES = 7;
36+
constexpr uint32_t NUM_BUCKETS = 2719;
37+
38+
std::vector<uint64_t> makeUInt64Keys(size_t size)
39+
{
40+
std::vector<uint64_t> keys;
41+
keys.reserve(size);
42+
for (size_t i = 0; i < size; ++i)
43+
keys.push_back(static_cast<uint64_t>(i * 0x9e3779b97f4a7c15ULL));
44+
return keys;
45+
}
46+
47+
std::vector<std::string> makeStringKeys(size_t size)
48+
{
49+
std::vector<std::string> keys;
50+
keys.reserve(size);
51+
for (size_t i = 0; i < size; ++i)
52+
keys.push_back("countmin-key-" + std::to_string(i * 2654435761ULL));
53+
return keys;
54+
}
55+
56+
int64_t totalStringBytes(const std::vector<std::string> & keys)
57+
{
58+
int64_t bytes = 0;
59+
for (const auto & key : keys)
60+
bytes += static_cast<int64_t>(key.size());
61+
return bytes;
62+
}
63+
64+
void BM_CountMinUpdateUInt64(benchmark::State & state)
65+
{
66+
const auto keys = makeUInt64Keys(static_cast<size_t>(state.range(0)));
67+
for (auto _ : state)
68+
{
69+
state.PauseTiming();
70+
Sketch sketch(NUM_HASHES, NUM_BUCKETS);
71+
state.ResumeTiming();
72+
73+
for (const auto key : keys)
74+
sketch.update(&key, sizeof(key), 1);
75+
76+
benchmark::ClobberMemory();
77+
benchmark::DoNotOptimize(sketch.get_total_weight());
78+
}
79+
80+
state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
81+
state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(keys.size() * sizeof(uint64_t)));
82+
}
83+
84+
void BM_CountMinUpdateStringBytes(benchmark::State & state)
85+
{
86+
const auto keys = makeStringKeys(static_cast<size_t>(state.range(0)));
87+
const auto bytes_per_iteration = totalStringBytes(keys);
88+
89+
for (auto _ : state)
90+
{
91+
state.PauseTiming();
92+
Sketch sketch(NUM_HASHES, NUM_BUCKETS);
93+
state.ResumeTiming();
94+
95+
for (const auto & key : keys)
96+
sketch.update(key.data(), key.size(), 1);
97+
98+
benchmark::ClobberMemory();
99+
benchmark::DoNotOptimize(sketch.get_total_weight());
100+
}
101+
102+
state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
103+
state.SetBytesProcessed(state.iterations() * bytes_per_iteration);
104+
}
105+
106+
void BM_CountMinEstimateUInt64(benchmark::State & state)
107+
{
108+
const auto keys = makeUInt64Keys(static_cast<size_t>(state.range(0)));
109+
Sketch sketch(NUM_HASHES, NUM_BUCKETS);
110+
for (const auto key : keys)
111+
sketch.update(&key, sizeof(key), 1);
112+
113+
uint64_t sum = 0;
114+
for (auto _ : state)
115+
{
116+
for (const auto key : keys)
117+
sum += sketch.get_estimate(&key, sizeof(key));
118+
119+
benchmark::DoNotOptimize(sum);
120+
}
121+
122+
state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
123+
state.SetBytesProcessed(state.iterations() * static_cast<int64_t>(keys.size() * sizeof(uint64_t)));
124+
}
125+
126+
void BM_CountMinEstimateStringBytes(benchmark::State & state)
127+
{
128+
const auto keys = makeStringKeys(static_cast<size_t>(state.range(0)));
129+
const auto bytes_per_iteration = totalStringBytes(keys);
130+
131+
Sketch sketch(NUM_HASHES, NUM_BUCKETS);
132+
for (const auto & key : keys)
133+
sketch.update(key.data(), key.size(), 1);
134+
135+
uint64_t sum = 0;
136+
for (auto _ : state)
137+
{
138+
for (const auto & key : keys)
139+
sum += sketch.get_estimate(key.data(), key.size());
140+
141+
benchmark::DoNotOptimize(sum);
142+
}
143+
144+
state.SetItemsProcessed(state.iterations() * static_cast<int64_t>(keys.size()));
145+
state.SetBytesProcessed(state.iterations() * bytes_per_iteration);
146+
}
147+
148+
}
149+
150+
BENCHMARK(BM_CountMinUpdateUInt64)->RangeMultiplier(8)->Range(1024, 65536);
151+
BENCHMARK(BM_CountMinUpdateStringBytes)->RangeMultiplier(8)->Range(1024, 65536);
152+
BENCHMARK(BM_CountMinEstimateUInt64)->RangeMultiplier(8)->Range(1024, 65536);
153+
BENCHMARK(BM_CountMinEstimateStringBytes)->RangeMultiplier(8)->Range(1024, 65536);

‎count/include/count_min.hpp‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -401,12 +401,13 @@ class count_min_sketch{
401401
static void check_header_validity(uint8_t preamble_longs, uint8_t serial_version, uint8_t family_id, uint8_t flags_byte);
402402

403403
/*
404-
* Obtain the hash values when inserting an item into the sketch.
405-
* @param item pointer to the data item to be inserted into the sketch.
404+
* Compute the hash locations for an input item
405+
* @param item pointer to the data item to be inserted into or queried from the sketch.
406406
* @param size of the data in bytes
407-
* @return vector of uint64_t which each represent the index to which `value' must update in the sketch
407+
* @param callback function to invoke for each sketch array location
408408
*/
409-
std::vector<uint64_t> get_hashes(const void* item, size_t size) const;
409+
template<typename F>
410+
void foreach_hash_location(const void* item, size_t size, F callback) const;
410411

411412
};
412413

‎count/include/count_min_impl.hpp‎

Lines changed: 11 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -110,9 +110,10 @@ uint8_t count_min_sketch<W,A>::suggest_num_hashes(double confidence) {
110110
}
111111

112112
template<typename W, typename A>
113-
std::vector<uint64_t> count_min_sketch<W,A>::get_hashes(const void* item, size_t size) const {
113+
template<typename F>
114+
void count_min_sketch<W,A>::foreach_hash_location(const void* item, size_t size, F callback) const {
114115
/*
115-
* Returns the hash locations for the input item using the original hashing
116+
* Computes the hash locations for the input item using the original hashing
116117
* scheme from [1].
117118
* Generate _num_hashes separate hashes from calls to murmurmhash.
118119
* This could be optimized by keeping both of the 64bit parts of the hash
@@ -126,19 +127,16 @@ std::vector<uint64_t> count_min_sketch<W,A>::get_hashes(const void* item, size_t
126127
* https://www.eecs.harvard.edu/~michaelm/postscripts/tr-02-05.pdf
127128
*/
128129
uint64_t bucket_index;
129-
std::vector<uint64_t> sketch_update_locations;
130-
sketch_update_locations.reserve(_num_hashes);
131130

132131
uint64_t hash_seed_index = 0;
133132
for (const auto &it: hash_seeds) {
134133
HashState hashes;
135134
MurmurHash3_x64_128(item, size, it, hashes); // ? BEWARE OVERFLOW.
136135
uint64_t hash = hashes.h1;
137136
bucket_index = hash % _num_buckets;
138-
sketch_update_locations.push_back((hash_seed_index * _num_buckets) + bucket_index);
137+
callback((hash_seed_index * _num_buckets) + bucket_index);
139138
hash_seed_index += 1;
140139
}
141-
return sketch_update_locations;
142140
}
143141

144142
template<typename W, typename A>
@@ -158,12 +156,11 @@ W count_min_sketch<W,A>::get_estimate(const void* item, size_t size) const {
158156
/*
159157
* Returns the estimated frequency of the item
160158
*/
161-
std::vector<uint64_t> hash_locations = get_hashes(item, size);
162-
std::vector<W> estimates;
163-
for (const auto h: hash_locations) {
164-
estimates.push_back(_sketch_array[h]);
165-
}
166-
return *std::min_element(estimates.begin(), estimates.end());
159+
W estimate = std::numeric_limits<W>::max();
160+
foreach_hash_location(item, size, [this, &estimate](uint64_t h) {
161+
estimate = std::min(estimate, _sketch_array[h]);
162+
});
163+
return estimate;
167164
}
168165

169166
template<typename W, typename A>
@@ -189,10 +186,9 @@ void count_min_sketch<W,A>::update(const void* item, size_t size, W weight) {
189186
* locations by the weight.
190187
*/
191188
_total_weight += weight >= 0 ? weight : -weight;
192-
std::vector<uint64_t> hash_locations = get_hashes(item, size);
193-
for (const auto h: hash_locations) {
189+
foreach_hash_location(item, size, [this, weight](uint64_t h) {
194190
_sketch_array[h] += weight;
195-
}
191+
});
196192
}
197193

198194
template<typename W, typename A>

0 commit comments

Comments
 (0)