perf: AVX2 vectorized prefix-sum in percentile scan (manually applied).

Author: mikeb01

src/hdr_histogram.c

@@ -34,6 +34,14 @@
 #  define HDR_UNLIKELY(x) (x)
 #endif
 
+/* Runtime-dispatched AVX2 path: keep the rest of this TU at the project's
+   baseline ISA so the shipped binary does not silently require AVX2. */
+#if (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) \
+    && (defined(__GNUC__) || defined(__clang__)) && !defined(__INTEL_COMPILER)
+#  define HDR_HAS_AVX2_DISPATCH 1
+#  include <immintrin.h>
+#endif
+
 /*  ######   #######  ##     ## ##    ## ########  ######  */
 /* ##    ## ##     ## ##     ## ###   ##    ##    ##    ## */
 /* ##       ##     ## ##     ## ####  ##    ##    ##       */
@@ -700,22 +708,65 @@ int64_t hdr_min(const struct hdr_histogram* h)
     return non_zero_min(h);
 }
 
-static int64_t get_value_from_idx_up_to_count(const struct hdr_histogram* h, int64_t count_at_percentile)
+static int64_t get_value_from_idx_up_to_count_scalar(
+    const struct hdr_histogram* h, int64_t count_at_percentile)
 {
     int64_t count_to_idx = 0;
-
-    count_at_percentile = 0 < count_at_percentile ? count_at_percentile : 1;
-    for (int32_t idx = 0; idx < h->counts_len; idx++)
-    {
+    for (int32_t idx = 0; idx < h->counts_len; idx++) {
         count_to_idx += h->counts[idx];
         if (count_to_idx >= count_at_percentile)
-        {
             return hdr_value_at_index(h, idx);
-        }
     }
+    return 0;
+}
 
+#ifdef HDR_HAS_AVX2_DISPATCH
+__attribute__((target("avx2")))
+static int64_t get_value_from_idx_up_to_count_avx2(
+    const struct hdr_histogram* h, int64_t count_at_percentile)
+{
+    int64_t running = 0;
+    int32_t idx = 0;
+    const int32_t limit = h->counts_len & ~3;
+
+    for (; idx < limit; idx += 4) {
+        __m256i v = _mm256_loadu_si256((const __m256i*)&h->counts[idx]);
+        __m128i lo = _mm256_castsi256_si128(v);
+        __m128i hi = _mm256_extracti128_si256(v, 1);
+        __m128i s = _mm_add_epi64(lo, hi);
+        /* Lanes are non-negative counts whose total fits in int64_t (total_count
+           invariant), so the chunk sum cannot overflow under valid state. Use
+           unsigned add to avoid signed-overflow UB if invariants are violated. */
+        int64_t chunk = (int64_t)((uint64_t)_mm_extract_epi64(s, 0)
+                                + (uint64_t)_mm_extract_epi64(s, 1));
+
+        if (__builtin_expect(running + chunk >= count_at_percentile, 0)) {
+            for (int32_t j = idx; j < idx + 4; j++) {
+                running += h->counts[j];
+                if (running >= count_at_percentile)
+                    return hdr_value_at_index(h, j);
+            }
+        }
+        running += chunk;
+    }
+    for (; idx < h->counts_len; idx++) {
+        running += h->counts[idx];
+        if (running >= count_at_percentile)
+            return hdr_value_at_index(h, idx);
+    }
     return 0;
 }
+#endif
+
+static int64_t get_value_from_idx_up_to_count(const struct hdr_histogram* h, int64_t count_at_percentile)
+{
+    count_at_percentile = count_at_percentile > 0 ? count_at_percentile : 1;
+#ifdef HDR_HAS_AVX2_DISPATCH
+    if (__builtin_cpu_supports("avx2"))
+        return get_value_from_idx_up_to_count_avx2(h, count_at_percentile);
+#endif
+    return get_value_from_idx_up_to_count_scalar(h, count_at_percentile);
+}
 
 
 int64_t hdr_value_at_percentile(const struct hdr_histogram* h, double percentile)

test/CMakeLists.txt

@@ -1,5 +1,8 @@
 option(HDR_HISTOGRAM_BUILD_BENCHMARK "Build benchmark" OFF)
 if(HDR_HISTOGRAM_BUILD_BENCHMARK)
+    add_executable(hdr_percentile_bench hdr_percentile_bench.c)
+    target_link_libraries(hdr_percentile_bench hdr_histogram_static)
+
     if(UNIX)
         enable_language(CXX)
 

test/hdr_percentile_bench.c

@@ -0,0 +1,56 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <hdr/hdr_histogram.h>
+#include <hdr/hdr_time.h>
+
+static hdr_timespec diff(hdr_timespec s, hdr_timespec e) {
+    hdr_timespec t;
+    if (e.tv_nsec < s.tv_nsec) { t.tv_sec = e.tv_sec-s.tv_sec-1; t.tv_nsec = 1000000000+e.tv_nsec-s.tv_nsec; }
+    else { t.tv_sec = e.tv_sec-s.tv_sec; t.tv_nsec = e.tv_nsec-s.tv_nsec; }
+    return t;
+}
+
+int main(void) {
+    struct hdr_histogram* h;
+    hdr_init(1, INT64_C(3600000000), 3, &h);
+
+    /* Spread non-zero entries across the full bucket range so the percentile
+       scan actually walks past the prologue. Fibonacci-hash spread (constant
+       2654435761) gives an even distribution over a coprime modulus. */
+    for (int64_t v = 1; v <= 1000000; v++) {
+        int64_t value = (int64_t)(((uint64_t)v * 2654435761u) % 1000000000u) + 1;
+        hdr_record_value(h, value);
+    }
+
+    const double percentiles[] = {50.0, 75.0, 90.0, 95.0, 99.0, 99.9, 99.99};
+    const int n_percentiles = (int)(sizeof(percentiles) / sizeof(percentiles[0]));
+    const int64_t iters = 1000000;
+    const int warmup_runs = 3;
+    const int n_runs = 20;
+
+    double best_secs = 1e18, total_secs = 0;
+    int64_t sink_total = 0;
+    for (int run = 0; run < warmup_runs + n_runs; run++) {
+        hdr_timespec t0, t1;
+        hdr_gettime(&t0);
+        volatile int64_t sink = 0;
+        for (int64_t i = 0; i < iters; i++)
+            sink += hdr_value_at_percentile(h, percentiles[i % n_percentiles]);
+        hdr_gettime(&t1);
+        hdr_timespec taken = diff(t0, t1);
+        double secs = taken.tv_sec + taken.tv_nsec / 1e9;
+        if (run >= warmup_runs) {
+            if (secs < best_secs) best_secs = secs;
+            total_secs += secs;
+            sink_total += sink;
+        }
+    }
+    double best_qps = iters / best_secs / 1e6;
+    double mean_qps = iters / (total_secs / n_runs) / 1e6;
+    printf("best: %.2f M queries/sec  mean: %.2f M queries/sec  (sink=%lld)\n",
+           best_qps, mean_qps, (long long)sink_total);
+
+    hdr_close(h);
+    return 0;
+}