DedupBench release 2.0

.github/workflows/makefile.yml

@@ -0,0 +1,28 @@
+name: Makefile CI
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@main
+
+    - name : Set up build environment
+      run: sudo apt install -y libssl-dev libxxhash-dev
+      working-directory: build
+
+    - name: Check base build
+      run: make all
+      working-directory: build
+
+    - name: Check SIMD build
+      run: make simd_all
+      working-directory: build
+

.gitignore

@@ -35,4 +35,8 @@
 .vscode/*
 
 # MacOS files
-**/.DS_Store
+**/.DS_Store
+
+results.txt
+results_*.txt
+results_graph.png

build/Makefile

@@ -1,4 +1,5 @@
-export ${EXTRA_COMPILER_FLAGS}
+# Note: The default build target 'all' does not use SIMD acceleration
+
 DEDUP_BUILD_PATH = ../dedup/build
 MEASURE_DEDUP_BUILD_PATH = ../measure-dedup/build
 MEASURE_LOW_ENTROPY_BUILD_PATH = ../measure-low-entropy/build
@@ -25,8 +26,41 @@ all:
 	cp $(SUPPORTING_TOOLS_PATH)/archive_extract.sh .
 	cp $(SUPPORTING_TOOLS_PATH)/archive_ctl_path.cfg .
 
+# To enable acceleration, pass the appropriate flags as EXTRA_COMPILER_FLAGS.
+# For SSE-128, pass '-msse -msse2 -msse3 -msse4.1' as EXTRA_COMPILER_FLAGS
+# For AVX-256, pass '-mavx -mavx2' as EXTRA_COMPILER_FLAGS
+# For BMI2, pass '-mbmi -mbmi2' as EXTRA_COMPILER_FLAGS (used only in VSEQ)
+# For AVX-512, pass '-mavx512f -mavx512vl -mavx512bw' as EXTRA_COMPILER_FLAGS
+
+
+.PHONY: sse128
+sse128:
+	$(MAKE) EXTRA_COMPILER_FLAGS="-msse -msse2 -msse3 -msse4.1" all
+
+.PHONY: avx256
+avx256:
+	$(MAKE) EXTRA_COMPILER_FLAGS="-mavx -mavx2" all
+
+.PHONY: avx512
+avx512:
+	$(MAKE) EXTRA_COMPILER_FLAGS="-mavx512f -mavx512vl -mavx512bw" all
+
+.PHONY: arm_neon128
+arm_neon128:
+	$(MAKE) all
 
+.PHONY: ibm_altivec128
+ibm_altivec128:
+	$(MAKE) EXTRA_COMPILER_FLAGS="-maltivec=le -mpowerpc64le" all
 
+.PHONY: simd_all
+simd_all:
+	$(MAKE) EXTRA_COMPILER_FLAGS="-msse -msse2 -msse3 -msse4.1 -mavx -mavx2 -mbmi -mbmi2" all
+
+.PHONY: simd512_all
+simd512_all:
+	$(MAKE) EXTRA_COMPILER_FLAGS="-msse -msse2 -msse3 -msse4.1 -mavx -mavx2 -mbmi -mbmi2 -mavx512f -mavx512vl -mavx512bw" all
+
 .PHONY: clean
 clean:
 	cd $(DEDUP_BUILD_PATH) && make $@

build/config.txt

@@ -35,6 +35,10 @@ ae_avg_block_size=8448
 ram_max_block_size=32768
 ram_avg_block_size=8448
 
+# MAXP Parameters
+maxp_window_size = 960
+maxp_max_block_size = 32768
+
 # TTTD Parameters
 tttd_min_block_size=4096
 tttd_avg_block_size=8192

build/config_simd512_8kb/aemax_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/aemax_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=max
+ae_avg_block_size=8448

build/config_simd512_8kb/aemin_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/aemin_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=min
+ae_avg_block_size=8448

build/config_simd512_8kb/crc32_8kb.conf

@@ -0,0 +1,15 @@
+# General Parameters
+chunking_algo = crc
+hashing_algo = xxhash128
+output_file = ./hashes_simd512_8kb/crc32_8kb.out
+buffer_size = 32768
+simd_mode=none
+
+# 8k Avg Chunk Size
+crc_hash_bits=13
+crc_window_size=256
+crc_window_step_size=1
+
+crc_min_block_size=1024
+crc_avg_block_size=8192
+crc_max_block_size=32768

build/config_simd512_8kb/fastcdc_8kb.conf

@@ -0,0 +1,14 @@
+# General Parameters
+chunking_algo=fastcdc
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/fastcdc_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# FastCDC Parameters
+fastcdc_min_block_size=2048
+fastcdc_avg_block_size=8192
+fastcdc_max_block_size=32768
+fastcdc_normalization_level=2
+fastcdc_disable_normalization=false

build/config_simd512_8kb/gear_8kb.conf

@@ -0,0 +1,12 @@
+# General Parameters
+chunking_algo=gear
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/gear_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# Gear Chunking parameters
+gear_min_block_size=2048
+gear_avg_block_size=8192
+gear_max_block_size=32768

build/config_simd512_8kb/maxp_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=maxp
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/maxp_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# MAXP Parameters
+maxp_window_size = 960
+maxp_max_block_size = 32768

build/config_simd512_8kb/rabins_8kb.conf

@@ -0,0 +1,13 @@
+# General Parameters
+chunking_algo=rabins
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/rabins_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# Rabin Chunking Parameters
+rabinc_min_block_size=2048
+rabinc_avg_block_size=8192
+rabinc_max_block_size=32768
+rabinc_window_size=48

build/config_simd512_8kb/ram_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ram
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/ram_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# RAM Parameters
+ram_max_block_size=32768
+ram_avg_block_size=8448

build/config_simd512_8kb/tttd_8kb.conf

@@ -0,0 +1,18 @@
+# General Parameters
+chunking_algo=tttd
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/tttd_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# TTTD Parameters
+tttd_min_block_size=2048
+tttd_avg_block_size=8192
+tttd_max_block_size=32768
+
+# Rabin Chunking Parameters
+rabinc_min_block_size=2048
+rabinc_avg_block_size=8192
+rabinc_max_block_size=32768
+rabinc_window_size=48

build/config_simd512_8kb/vaemax512_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/vaemax512_8kb.out
+simd_mode=avx512
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=max
+ae_avg_block_size=8448

build/config_simd512_8kb/vaemin512_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/vaemin512_8kb.out
+simd_mode=avx512
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=min
+ae_avg_block_size=8448

build/config_simd512_8kb/vmaxp512_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=maxp
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/vmaxp512_8kb.out
+simd_mode=avx512
+
+buffer_size=32768
+
+# MAXP Parameters
+maxp_window_size = 960
+maxp_max_block_size = 32768

build/config_simd512_8kb/vram512_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ram
+hashing_algo=xxhash128
+output_file=./hashes_simd512_8kb/vram512_8kb.out
+simd_mode=avx512
+
+buffer_size=32768
+
+# RAM Parameters
+ram_max_block_size=32768
+ram_avg_block_size=8448

build/config_simd_8kb copy/aemax_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/aemax_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=max
+ae_avg_block_size=8448

build/config_simd_8kb copy/aemin_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ae
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/aemin_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# AE Parameters
+ae_extreme_mode=min
+ae_avg_block_size=8448

build/config_simd_8kb copy/crc32_8kb.conf

@@ -0,0 +1,15 @@
+# General Parameters
+chunking_algo = crc
+hashing_algo = xxhash128
+output_file = ./hashes_simd_8kb/crc32_8kb.out
+buffer_size = 32768
+simd_mode=none
+
+# 8k Avg Chunk Size
+crc_hash_bits=13
+crc_window_size=256
+crc_window_step_size=1
+
+crc_min_block_size=1024
+crc_avg_block_size=8192
+crc_max_block_size=32768

build/config_simd_8kb copy/fastcdc_8kb.conf

@@ -0,0 +1,14 @@
+# General Parameters
+chunking_algo=fastcdc
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/fastcdc_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# FastCDC Parameters
+fastcdc_min_block_size=2048
+fastcdc_avg_block_size=8192
+fastcdc_max_block_size=32768
+fastcdc_normalization_level=2
+fastcdc_disable_normalization=false

build/config_simd_8kb copy/gear_8kb.conf

@@ -0,0 +1,12 @@
+# General Parameters
+chunking_algo=gear
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/gear_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# Gear Chunking parameters
+gear_min_block_size=2048
+gear_avg_block_size=8192
+gear_max_block_size=32768

build/config_simd_8kb copy/maxp_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=maxp
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/maxp_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# MAXP Parameters
+maxp_window_size = 960
+maxp_max_block_size = 32768

build/config_simd_8kb copy/rabins_8kb.conf

@@ -0,0 +1,13 @@
+# General Parameters
+chunking_algo=rabins
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/rabins_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# Rabin Chunking Parameters
+rabinc_min_block_size=2048
+rabinc_avg_block_size=8192
+rabinc_max_block_size=32768
+rabinc_window_size=48

build/config_simd_8kb copy/ram_8kb.conf

@@ -0,0 +1,11 @@
+# General Parameters
+chunking_algo=ram
+hashing_algo=xxhash128
+output_file=./hashes_simd_8kb/ram_8kb.out
+simd_mode=none
+
+buffer_size=32768
+
+# RAM Parameters
+ram_max_block_size=32768
+ram_avg_block_size=8448