optimize crc32c_hw_series() (alibaba#971)

Author: lihuiba

common/checksum/crc.cpp

@@ -690,23 +690,37 @@ void crc32c_series_sw(const uint8_t *buffer, uint32_t part_size, uint32_t n_part
 }
 
 void crc32c_series_hw(const uint8_t *buffer, uint32_t part_size, uint32_t n_parts, uint32_t* crc_parts) {
-    memset(crc_parts, 0, n_parts * sizeof(uint32_t));
-    uint64_t i = 0;
-    auto perf = [&](auto type) {
-        using T = decltype(type);
-        for (; i < part_size; i += sizeof(T)) {
-            #pragma GCC unroll 8
-            for (uint64_t j = 0; j < n_parts; ++j) {
-                auto ptr = buffer + i + j * part_size;
-                crc_parts[j] = crc32c(crc_parts[j], *(T*)ptr);
-            }
+    const size_t BATCH = 4;
+    auto part_main = part_size / 8 * 8;
+    auto part_remain = part_size % 8;
+    auto batch_crc = [&](auto ptr, size_t batch) __attribute__((always_inline)) {
+        #pragma GCC unroll 4
+        for (size_t k = 0; k < batch; ++k) {
+            crc_parts[k] = crc32c(crc_parts[k], *ptr);
+            ptr = decltype(ptr)((char*)ptr + part_size);
         }
     };
-    perf(i);
-    perf(buffer[0]);
+    auto batch_blocks_crc = [&](const uint8_t* ptr, auto batch) __attribute__((always_inline)) {
+        for (size_t j = 0; j < batch; ++j)
+            crc_parts[j] = 0;
+        for (; ptr < buffer + part_main; ptr += 8) {
+            batch_crc((uint64_t*)ptr, batch);
+        }
+        if (unlikely(part_main)) {
+            if (part_remain & 4) { batch_crc((uint32_t*)ptr, batch); ptr += 4; }
+            if (part_remain & 2) { batch_crc((uint16_t*)ptr, batch); ptr += 2; }
+            if (part_remain & 1) { batch_crc((uint8_t*)ptr, batch); ptr += 1; }
+        }
+    };
+    for (; n_parts >= BATCH; n_parts -= BATCH) {
+        batch_blocks_crc(buffer, BATCH);
+        buffer += part_size * BATCH;
+        crc_parts += BATCH;
+    }
+    if (unlikely(n_parts))
+        batch_blocks_crc(buffer, n_parts);
 }
 
-
 // rk1 ~ rk20
 __attribute__((aligned(16), used))
 const static uint64_t rk[20] = {

common/checksum/test/test_checksum.cpp

@@ -143,6 +143,7 @@ inline void do_perf_crc(const char* name, CRC64ECMA crc64ecma, unsigned long siz
 }
 
 const size_t _128KB = 128 * 1024;
+const size_t _1MB = 1024 * 1024;
 const size_t _1GB = 1024 * 1024 * 1024;
 
 TEST(Perf, crc32c_hw_simple) {
@@ -252,6 +253,27 @@ TEST(TestChecksum, crc32_combine) {
     EXPECT_EQ(crc32c_trim({x, sizeof(buf)}, {crc1, 128}, {crc3, 128}), crc2);
 }
 
+void crc32c_series_hw_old(const uint8_t *buffer, uint32_t part_size, uint32_t n_parts, uint32_t* crc_parts);
+
+TEST(Perf, crc32c_hw_series) {
+    const uint32_t PART_SIZE = 4096;
+    static uint32_t crc[1024*1024];
+    auto f = [](const uint8_t *buffer, size_t nbytes, uint32_t) -> uint32_t {
+        crc32c_series_hw(buffer, PART_SIZE, nbytes / PART_SIZE, crc);
+        return 0;
+    };
+    do_perf_crc("crc32c_hw_series", f, _128KB);
+    do_perf_crc("crc32c_hw_series", f, _1MB);
+
+    auto f_naive = [](const uint8_t *buffer, size_t nbytes, uint32_t) -> uint32_t {
+        for (size_t i = 0; i < nbytes / PART_SIZE; ++i)
+            crc[i] = crc32c_hw(buffer + i * PART_SIZE, PART_SIZE, 0);
+        return 0;
+    };
+    do_perf_crc("crc32c_hw_series_naive", f_naive, _128KB);
+    do_perf_crc("crc32c_hw_series_naive", f_naive, _1MB);
+}
+
 int main(int argc, char **argv)
 {
     if (!photon::is_using_default_engine()) return 0;