@@ -10,25 +10,27 @@ use crate::core::Fingerprint;
1010use crate :: FINGERPRINT_U64 ;
1111
1212
13- #[ cfg( target_arch = "aarch64" ) ]
14- use std:: arch:: aarch64:: * ;
15-
1613/// Compute Hamming distance between two fingerprints.
1714///
1815/// Automatically dispatches to the best SIMD implementation available.
1916#[ inline]
2017pub fn hamming_distance ( a : & Fingerprint , b : & Fingerprint ) -> u32 {
2118 #[ cfg( all( target_arch = "x86_64" , target_feature = "avx512vpopcntdq" ) ) ]
22- unsafe { return hamming_avx512 ( a, b) ; }
23-
19+ { return unsafe { hamming_avx512 ( a, b) } ; }
20+
2421 #[ cfg( all( target_arch = "x86_64" , target_feature = "avx2" , not( target_feature = "avx512vpopcntdq" ) ) ) ]
25- unsafe { return hamming_avx2 ( a, b) ; }
26-
22+ { return unsafe { hamming_avx2 ( a, b) } ; }
23+
2724 #[ cfg( all( target_arch = "aarch64" , target_feature = "neon" ) ) ]
28- unsafe { return hamming_neon ( a, b) ; }
29-
25+ { return unsafe { hamming_neon ( a, b) } ; }
26+
3027 // Scalar fallback
31- hamming_scalar ( a, b)
28+ #[ cfg( not( any(
29+ all( target_arch = "x86_64" , target_feature = "avx512vpopcntdq" ) ,
30+ all( target_arch = "x86_64" , target_feature = "avx2" ) ,
31+ all( target_arch = "aarch64" , target_feature = "neon" ) ,
32+ ) ) ) ]
33+ { return hamming_scalar ( a, b) ; }
3234}
3335
3436/// Scalar implementation (works everywhere)
@@ -48,123 +50,135 @@ pub fn hamming_scalar(a: &Fingerprint, b: &Fingerprint) -> u32 {
4850#[ cfg( all( target_arch = "x86_64" , target_feature = "avx512vpopcntdq" ) ) ]
4951#[ target_feature( enable = "avx512f" , enable = "avx512vpopcntdq" ) ]
5052unsafe fn hamming_avx512 ( a : & Fingerprint , b : & Fingerprint ) -> u32 {
51- let a_ptr = a. as_raw ( ) . as_ptr ( ) ;
52- let b_ptr = b. as_raw ( ) . as_ptr ( ) ;
53-
54- let mut sum = _mm512_setzero_si512 ( ) ;
55-
56- // Process 8 u64 at a time (512 bits)
57- let mut i = 0 ;
58- while i + 8 <= FINGERPRINT_U64 {
59- let va = _mm512_loadu_si512 ( a_ptr. add ( i) as * const __m512i ) ;
60- let vb = _mm512_loadu_si512 ( b_ptr. add ( i) as * const __m512i ) ;
61- let xor = _mm512_xor_si512 ( va, vb) ;
62- let popcnt = _mm512_popcnt_epi64 ( xor) ;
63- sum = _mm512_add_epi64 ( sum, popcnt) ;
64- i += 8 ;
65- }
66-
67- // Horizontal sum
68- let mut total = _mm512_reduce_add_epi64 ( sum) as u32 ;
69-
70- // Handle remaining (256 % 8 = 0, no remainder at 16K)
71- while i < FINGERPRINT_U64 {
72- total += ( * a_ptr. add ( i) ^ * b_ptr. add ( i) ) . count_ones ( ) ;
73- i += 1 ;
53+ unsafe {
54+ use std:: arch:: x86_64:: * ;
55+
56+ let a_ptr = a. as_raw ( ) . as_ptr ( ) ;
57+ let b_ptr = b. as_raw ( ) . as_ptr ( ) ;
58+
59+ let mut sum = _mm512_setzero_si512 ( ) ;
60+
61+ // Process 8 u64 at a time (512 bits)
62+ let mut i = 0 ;
63+ while i + 8 <= FINGERPRINT_U64 {
64+ let va = _mm512_loadu_si512 ( a_ptr. add ( i) as * const __m512i ) ;
65+ let vb = _mm512_loadu_si512 ( b_ptr. add ( i) as * const __m512i ) ;
66+ let xor = _mm512_xor_si512 ( va, vb) ;
67+ let popcnt = _mm512_popcnt_epi64 ( xor) ;
68+ sum = _mm512_add_epi64 ( sum, popcnt) ;
69+ i += 8 ;
70+ }
71+
72+ // Horizontal sum
73+ let mut total = _mm512_reduce_add_epi64 ( sum) as u32 ;
74+
75+ // Handle remaining (256 % 8 = 0, no remainder at 16K)
76+ while i < FINGERPRINT_U64 {
77+ total += ( * a_ptr. add ( i) ^ * b_ptr. add ( i) ) . count_ones ( ) ;
78+ i += 1 ;
79+ }
80+
81+ total
7482 }
75-
76- total
7783}
7884
7985/// AVX2 implementation (fallback for older x86_64)
8086#[ cfg( all( target_arch = "x86_64" , target_feature = "avx2" ) ) ]
8187#[ target_feature( enable = "avx2" ) ]
8288unsafe fn hamming_avx2 ( a : & Fingerprint , b : & Fingerprint ) -> u32 {
83- let a_ptr = a. as_raw ( ) . as_ptr ( ) ;
84- let b_ptr = b. as_raw ( ) . as_ptr ( ) ;
85-
86- // Lookup table for 4-bit popcount
87- let lookup = _mm256_setr_epi8 (
88- 0 , 1 , 1 , 2 , 1 , 2 , 2 , 3 , 1 , 2 , 2 , 3 , 2 , 3 , 3 , 4 ,
89- 0 , 1 , 1 , 2 , 1 , 2 , 2 , 3 , 1 , 2 , 2 , 3 , 2 , 3 , 3 , 4 ,
90- ) ;
91- let low_mask = _mm256_set1_epi8 ( 0x0f ) ;
92-
93- let mut total_sum = _mm256_setzero_si256 ( ) ;
94-
95- // Process 4 u64 at a time (256 bits)
96- let mut i = 0 ;
97- while i + 4 <= FINGERPRINT_U64 {
98- let va = _mm256_loadu_si256 ( a_ptr. add ( i) as * const __m256i ) ;
99- let vb = _mm256_loadu_si256 ( b_ptr. add ( i) as * const __m256i ) ;
100- let xor = _mm256_xor_si256 ( va, vb) ;
101-
102- // Popcount via lookup table
103- let lo = _mm256_and_si256 ( xor, low_mask) ;
104- let hi = _mm256_and_si256 ( _mm256_srli_epi16 ( xor, 4 ) , low_mask) ;
105- let popcnt_lo = _mm256_shuffle_epi8 ( lookup, lo) ;
106- let popcnt_hi = _mm256_shuffle_epi8 ( lookup, hi) ;
107- let popcnt = _mm256_add_epi8 ( popcnt_lo, popcnt_hi) ;
108-
109- // Sum bytes
110- let sad = _mm256_sad_epu8 ( popcnt, _mm256_setzero_si256 ( ) ) ;
111- total_sum = _mm256_add_epi64 ( total_sum, sad) ;
112-
113- i += 4 ;
114- }
115-
116- // Horizontal sum
117- let sum_lo = _mm256_extracti128_si256 ( total_sum, 0 ) ;
118- let sum_hi = _mm256_extracti128_si256 ( total_sum, 1 ) ;
119- let sum128 = _mm_add_epi64 ( sum_lo, sum_hi) ;
120- let mut total = ( _mm_extract_epi64 ( sum128, 0 ) + _mm_extract_epi64 ( sum128, 1 ) ) as u32 ;
121-
122- // Handle remaining
123- while i < FINGERPRINT_U64 {
124- total += ( * a_ptr. add ( i) ^ * b_ptr. add ( i) ) . count_ones ( ) ;
125- i += 1 ;
89+ unsafe {
90+ use std:: arch:: x86_64:: * ;
91+
92+ let a_ptr = a. as_raw ( ) . as_ptr ( ) ;
93+ let b_ptr = b. as_raw ( ) . as_ptr ( ) ;
94+
95+ // Lookup table for 4-bit popcount
96+ let lookup = _mm256_setr_epi8 (
97+ 0 , 1 , 1 , 2 , 1 , 2 , 2 , 3 , 1 , 2 , 2 , 3 , 2 , 3 , 3 , 4 ,
98+ 0 , 1 , 1 , 2 , 1 , 2 , 2 , 3 , 1 , 2 , 2 , 3 , 2 , 3 , 3 , 4 ,
99+ ) ;
100+ let low_mask = _mm256_set1_epi8 ( 0x0f ) ;
101+
102+ let mut total_sum = _mm256_setzero_si256 ( ) ;
103+
104+ // Process 4 u64 at a time (256 bits)
105+ let mut i = 0 ;
106+ while i + 4 <= FINGERPRINT_U64 {
107+ let va = _mm256_loadu_si256 ( a_ptr. add ( i) as * const __m256i ) ;
108+ let vb = _mm256_loadu_si256 ( b_ptr. add ( i) as * const __m256i ) ;
109+ let xor = _mm256_xor_si256 ( va, vb) ;
110+
111+ // Popcount via lookup table
112+ let lo = _mm256_and_si256 ( xor, low_mask) ;
113+ let hi = _mm256_and_si256 ( _mm256_srli_epi16 ( xor, 4 ) , low_mask) ;
114+ let popcnt_lo = _mm256_shuffle_epi8 ( lookup, lo) ;
115+ let popcnt_hi = _mm256_shuffle_epi8 ( lookup, hi) ;
116+ let popcnt = _mm256_add_epi8 ( popcnt_lo, popcnt_hi) ;
117+
118+ // Sum bytes
119+ let sad = _mm256_sad_epu8 ( popcnt, _mm256_setzero_si256 ( ) ) ;
120+ total_sum = _mm256_add_epi64 ( total_sum, sad) ;
121+
122+ i += 4 ;
123+ }
124+
125+ // Horizontal sum
126+ let sum_lo = _mm256_extracti128_si256 ( total_sum, 0 ) ;
127+ let sum_hi = _mm256_extracti128_si256 ( total_sum, 1 ) ;
128+ let sum128 = _mm_add_epi64 ( sum_lo, sum_hi) ;
129+ let mut total = ( _mm_extract_epi64 ( sum128, 0 ) + _mm_extract_epi64 ( sum128, 1 ) ) as u32 ;
130+
131+ // Handle remaining
132+ while i < FINGERPRINT_U64 {
133+ total += ( * a_ptr. add ( i) ^ * b_ptr. add ( i) ) . count_ones ( ) ;
134+ i += 1 ;
135+ }
136+
137+ total
126138 }
127-
128- total
129139}
130140
131141/// ARM NEON implementation
132142#[ cfg( all( target_arch = "aarch64" , target_feature = "neon" ) ) ]
133143#[ target_feature( enable = "neon" ) ]
134144unsafe fn hamming_neon ( a : & Fingerprint , b : & Fingerprint ) -> u32 {
135- let a_ptr = a. as_raw ( ) . as_ptr ( ) as * const u8 ;
136- let b_ptr = b. as_raw ( ) . as_ptr ( ) as * const u8 ;
137-
138- let mut sum = vdupq_n_u64 ( 0 ) ;
139-
140- // Process 16 bytes at a time
141- let mut i = 0 ;
142- let byte_len = FINGERPRINT_U64 * 8 ;
143- while i + 16 <= byte_len {
144- let va = vld1q_u8 ( a_ptr. add ( i) ) ;
145- let vb = vld1q_u8 ( b_ptr. add ( i) ) ;
146- let xor = veorq_u8 ( va, vb) ;
147- let cnt = vcntq_u8 ( xor) ; // Count bits per byte
148-
149- // Sum to 64-bit
150- let sum16 = vpaddlq_u8 ( cnt) ; // u8 -> u16
151- let sum32 = vpaddlq_u16 ( sum16) ; // u16 -> u32
152- let sum64 = vpaddlq_u32 ( sum32) ; // u32 -> u64
153- sum = vaddq_u64 ( sum, sum64) ;
154-
155- i += 16 ;
156- }
157-
158- // Horizontal sum
159- let mut total = ( vgetq_lane_u64 ( sum, 0 ) + vgetq_lane_u64 ( sum, 1 ) ) as u32 ;
160-
161- // Handle remaining bytes
162- while i < byte_len {
163- total += ( * a_ptr. add ( i) ^ * b_ptr. add ( i) ) . count_ones ( ) ;
164- i += 1 ;
145+ unsafe {
146+ use std:: arch:: aarch64:: * ;
147+
148+ let a_ptr = a. as_raw ( ) . as_ptr ( ) as * const u8 ;
149+ let b_ptr = b. as_raw ( ) . as_ptr ( ) as * const u8 ;
150+
151+ let mut sum = vdupq_n_u64 ( 0 ) ;
152+
153+ // Process 16 bytes at a time
154+ let mut i = 0 ;
155+ let byte_len = FINGERPRINT_U64 * 8 ;
156+ while i + 16 <= byte_len {
157+ let va = vld1q_u8 ( a_ptr. add ( i) ) ;
158+ let vb = vld1q_u8 ( b_ptr. add ( i) ) ;
159+ let xor = veorq_u8 ( va, vb) ;
160+ let cnt = vcntq_u8 ( xor) ; // Count bits per byte
161+
162+ // Sum to 64-bit
163+ let sum16 = vpaddlq_u8 ( cnt) ; // u8 -> u16
164+ let sum32 = vpaddlq_u16 ( sum16) ; // u16 -> u32
165+ let sum64 = vpaddlq_u32 ( sum32) ; // u32 -> u64
166+ sum = vaddq_u64 ( sum, sum64) ;
167+
168+ i += 16 ;
169+ }
170+
171+ // Horizontal sum
172+ let mut total = ( vgetq_lane_u64 ( sum, 0 ) + vgetq_lane_u64 ( sum, 1 ) ) as u32 ;
173+
174+ // Handle remaining bytes
175+ while i < byte_len {
176+ total += ( * a_ptr. add ( i) ^ * b_ptr. add ( i) ) . count_ones ( ) ;
177+ i += 1 ;
178+ }
179+
180+ total
165181 }
166-
167- total
168182}
169183
170184/// Batch Hamming distance computation (parallel)
0 commit comments