From e65f561f32e01a124e38ec2624b0e0c9d4cebec7 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 2 Jul 2026 19:58:23 +0100 Subject: [PATCH 1/6] Implement swizzle_dyn_within_blocks, add tests --- fearless_simd/src/generated/avx2.rs | 326 ++++++++++ fearless_simd/src/generated/avx512.rs | 385 +++++++++++ fearless_simd/src/generated/fallback.rs | 748 ++++++++++++++++++++++ fearless_simd/src/generated/neon.rs | 284 ++++++++ fearless_simd/src/generated/simd_trait.rs | 80 +++ fearless_simd/src/generated/simd_types.rs | 144 +++++ fearless_simd/src/generated/sse4_2.rs | 284 ++++++++ fearless_simd/src/generated/wasm.rs | 236 +++++++ fearless_simd_gen/src/generic.rs | 14 + fearless_simd_gen/src/mk_fallback.rs | 32 + fearless_simd_gen/src/mk_neon.rs | 20 + fearless_simd_gen/src/mk_simd_trait.rs | 2 + fearless_simd_gen/src/mk_simd_types.rs | 8 + fearless_simd_gen/src/mk_wasm.rs | 20 + fearless_simd_gen/src/mk_x86.rs | 38 ++ fearless_simd_gen/src/ops.rs | 23 + fearless_simd_gen/src/types.rs | 4 + fearless_simd_tests/tests/harness/mod.rs | 391 +++++++++++ 18 files changed, 3039 insertions(+) diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 3f4ee93e7..c306f88ed 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -175,6 +175,20 @@ impl Simd for Avx2 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x4, indices: u8x16) -> f32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -697,6 +711,20 @@ impl Simd for Avx2 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x16, indices: u8x16) -> i8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1065,6 +1093,20 @@ impl Simd for Avx2 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x16, indices: u8x16) -> u8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1606,6 +1648,20 @@ impl Simd for Avx2 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x8, indices: u8x16) -> i16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -1955,6 +2011,20 @@ impl Simd for Avx2 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x8, indices: u8x16) -> u16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2475,6 +2545,20 @@ impl Simd for Avx2 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x4, indices: u8x16) -> i32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -2844,6 +2928,20 @@ impl Simd for Avx2 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x4, indices: u8x16) -> u32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3380,6 +3478,20 @@ impl Simd for Avx2 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2, indices: u8x16) -> f64x2 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -3973,6 +4085,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8, indices: u8x32) -> f32x8 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_f32x8(a).val.0, indices.into()); + token.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -4558,6 +4684,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x32, indices: u8x32) -> i8x32 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_i8x32(a).val.0, indices.into()); + token.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -5014,6 +5154,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, indices: u8x32) -> u8x32 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_u8x32(a).val.0, indices.into()); + token.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -5657,6 +5811,25 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x16, indices: u8x32) -> i16x16 { + let result = + _mm256_shuffle_epi8(token.cvt_to_bytes_i16x16(a).val.0, indices.into()); + token.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -6096,6 +6269,25 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x16, indices: u8x32) -> u16x16 { + let result = + _mm256_shuffle_epi8(token.cvt_to_bytes_u16x16(a).val.0, indices.into()); + token.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -6737,6 +6929,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x8, indices: u8x32) -> i32x8 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_i32x8(a).val.0, indices.into()); + token.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -7172,6 +7378,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x8, indices: u8x32) -> u32x8 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_u32x8(a).val.0, indices.into()); + token.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -7786,6 +8006,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, indices: u8x32) -> f64x4 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_f64x4(a).val.0, indices.into()); + token.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -8420,6 +8654,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -8859,6 +9106,15 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -9130,6 +9386,15 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -9628,6 +9893,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -9908,6 +10186,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -10414,6 +10705,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -10690,6 +10994,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -11166,6 +11483,15 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 976ebd2ad..d2e015fe4 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -183,6 +183,21 @@ impl Simd for Avx512 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = token.cvt_to_bytes_f32x4(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -702,6 +717,21 @@ impl Simd for Avx512 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = token.cvt_to_bytes_i8x16(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1148,6 +1178,21 @@ impl Simd for Avx512 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = token.cvt_to_bytes_u8x16(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1719,6 +1764,21 @@ impl Simd for Avx512 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = token.cvt_to_bytes_i16x8(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2116,6 +2176,21 @@ impl Simd for Avx512 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = token.cvt_to_bytes_u16x8(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2638,6 +2713,21 @@ impl Simd for Avx512 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = token.cvt_to_bytes_i32x4(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3035,6 +3125,21 @@ impl Simd for Avx512 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = token.cvt_to_bytes_u32x4(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3548,6 +3653,21 @@ impl Simd for Avx512 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = token.cvt_to_bytes_f64x2(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4148,6 +4268,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = token.cvt_to_bytes_f32x8(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -4768,6 +4903,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = token.cvt_to_bytes_i8x32(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -5325,6 +5475,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = token.cvt_to_bytes_u8x32(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -6017,6 +6182,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = token.cvt_to_bytes_i16x16(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -6505,6 +6689,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = token.cvt_to_bytes_u16x16(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -7142,6 +7345,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = token.cvt_to_bytes_i32x8(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -7618,6 +7836,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = token.cvt_to_bytes_u32x8(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -8224,6 +8457,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = token.cvt_to_bytes_f64x4(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -8890,6 +9138,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, indices: u8x64) -> f32x16 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_f32x16(a).val.0, indices.into()); + token.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -9554,6 +9821,20 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, indices: u8x64) -> i8x64 { + let result = _mm512_shuffle_epi8(token.cvt_to_bytes_i8x64(a).val.0, indices.into()); + token.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { crate::kernel!( #[inline(always)] @@ -10127,6 +10408,20 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, indices: u8x64) -> u8x64 { + let result = _mm512_shuffle_epi8(token.cvt_to_bytes_u8x64(a).val.0, indices.into()); + token.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { crate::kernel!( #[inline(always)] @@ -10849,6 +11144,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, indices: u8x64) -> i16x32 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_i16x32(a).val.0, indices.into()); + token.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { crate::kernel!( #[inline(always)] @@ -11347,6 +11661,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, indices: u8x64) -> u16x32 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_u16x32(a).val.0, indices.into()); + token.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { crate::kernel!( #[inline(always)] @@ -12022,6 +12355,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, indices: u8x64) -> i32x16 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_i32x16(a).val.0, indices.into()); + token.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { crate::kernel!( #[inline(always)] @@ -12512,6 +12864,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, indices: u8x64) -> u32x16 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_u32x16(a).val.0, indices.into()); + token.cvt_from_bytes_u32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -13153,6 +13524,20 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, indices: u8x64) -> f64x8 { + let result = _mm512_shuffle_epi8(token.cvt_to_bytes_f64x8(a).val.0, indices.into()); + token.cvt_from_bytes_f64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 1024b172a..34f28fcdb 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -215,6 +215,78 @@ impl Simd for Fallback { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = self.cvt_to_bytes_f32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x4(result) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { [ f32::abs(a[0usize]), @@ -611,6 +683,78 @@ impl Simd for Fallback { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = self.cvt_to_bytes_i8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x16(result) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { [ i8::wrapping_add(a[0usize], b[0usize]), @@ -1236,6 +1380,78 @@ impl Simd for Fallback { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = self.cvt_to_bytes_u8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x16(result) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { [ u8::wrapping_add(a[0usize], b[0usize]), @@ -2187,6 +2403,78 @@ impl Simd for Fallback { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = self.cvt_to_bytes_i16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x8(result) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { [ i16::wrapping_add(a[0usize], b[0usize]), @@ -2613,6 +2901,78 @@ impl Simd for Fallback { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = self.cvt_to_bytes_u16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x8(result) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { [ u16::wrapping_add(a[0usize], b[0usize]), @@ -3243,6 +3603,78 @@ impl Simd for Fallback { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = self.cvt_to_bytes_i32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x4(result) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { [ i32::wrapping_add(a[0usize], b[0usize]), @@ -3571,6 +4003,78 @@ impl Simd for Fallback { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = self.cvt_to_bytes_u32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x4(result) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { [ u32::wrapping_add(a[0usize], b[0usize]), @@ -4035,6 +4539,78 @@ impl Simd for Fallback { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = self.cvt_to_bytes_f64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x2(result) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { [f64::abs(a[0usize]), f64::abs(a[1usize])].simd_into(self) } @@ -4441,6 +5017,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4798,6 +5383,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -5062,6 +5656,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -5440,6 +6043,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5704,6 +6320,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -6104,6 +6733,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -6373,6 +7011,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6748,6 +7395,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -7177,6 +7833,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -7557,6 +8226,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -7814,6 +8492,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -8258,6 +8945,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -8524,6 +9224,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -8953,6 +9666,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -9215,6 +9941,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -9606,6 +10345,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 8553ff661..3af220205 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -165,6 +165,20 @@ impl Simd for Neon { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x4, indices: u8x16) -> f32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -642,6 +656,20 @@ impl Simd for Neon { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x16, indices: u8x16) -> i8x16 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1008,6 +1036,20 @@ impl Simd for Neon { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x16, indices: u8x16) -> u8x16 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1553,6 +1595,20 @@ impl Simd for Neon { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x8, indices: u8x16) -> i16x8 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -1919,6 +1975,20 @@ impl Simd for Neon { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x8, indices: u8x16) -> u16x8 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2454,6 +2524,20 @@ impl Simd for Neon { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x4, indices: u8x16) -> i32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -2830,6 +2914,20 @@ impl Simd for Neon { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x4, indices: u8x16) -> u32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3364,6 +3462,20 @@ impl Simd for Neon { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x2, indices: u8x16) -> f64x2 { + let result = vqtbl1q_u8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -3981,6 +4093,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4375,6 +4496,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4676,6 +4806,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -5098,6 +5237,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5399,6 +5551,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -5834,6 +5999,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -6140,6 +6314,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6559,6 +6742,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -7050,6 +7242,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -7461,6 +7666,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -7771,6 +7985,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -8196,6 +8419,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -8515,6 +8751,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -8962,6 +9211,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -9277,6 +9539,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -9704,6 +9979,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index 7690bca33..fd6d09314 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -144,6 +144,8 @@ pub trait Simd: a: f32x4, b: f32x4, ) -> f32x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; #[doc = "Compute the absolute value of each element."] fn abs_f32x4(self, a: f32x4) -> f32x4; #[doc = "Negate each element of the vector."] @@ -252,6 +254,8 @@ pub trait Simd: a: i8x16, b: i8x16, ) -> i8x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -336,6 +340,8 @@ pub trait Simd: a: u8x16, b: u8x16, ) -> u8x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -457,6 +463,8 @@ pub trait Simd: a: i16x8, b: i16x8, ) -> i16x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -541,6 +549,8 @@ pub trait Simd: a: u16x8, b: u16x8, ) -> u16x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -662,6 +672,8 @@ pub trait Simd: a: i32x4, b: i32x4, ) -> i32x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -748,6 +760,8 @@ pub trait Simd: a: u32x4, b: u32x4, ) -> u32x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -869,6 +883,8 @@ pub trait Simd: a: f64x2, b: f64x2, ) -> f64x2; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; #[doc = "Compute the absolute value of each element."] fn abs_f64x2(self, a: f64x2) -> f64x2; #[doc = "Negate each element of the vector."] @@ -1002,6 +1018,8 @@ pub trait Simd: a: f32x8, b: f32x8, ) -> f32x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; #[doc = "Compute the absolute value of each element."] fn abs_f32x8(self, a: f32x8) -> f32x8; #[doc = "Negate each element of the vector."] @@ -1112,6 +1130,8 @@ pub trait Simd: a: i8x32, b: i8x32, ) -> i8x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1198,6 +1218,8 @@ pub trait Simd: a: u8x32, b: u8x32, ) -> u8x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1323,6 +1345,12 @@ pub trait Simd: a: i16x16, b: i16x16, ) -> i16x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1409,6 +1437,12 @@ pub trait Simd: a: u16x16, b: u16x16, ) -> u16x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1536,6 +1570,8 @@ pub trait Simd: a: i32x8, b: i32x8, ) -> i32x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1624,6 +1660,8 @@ pub trait Simd: a: u32x8, b: u32x8, ) -> u32x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1749,6 +1787,8 @@ pub trait Simd: a: f64x4, b: f64x4, ) -> f64x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; #[doc = "Compute the absolute value of each element."] fn abs_f64x4(self, a: f64x4) -> f64x4; #[doc = "Negate each element of the vector."] @@ -1886,6 +1926,12 @@ pub trait Simd: a: f32x16, b: f32x16, ) -> f32x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16; #[doc = "Compute the absolute value of each element."] fn abs_f32x16(self, a: f32x16) -> f32x16; #[doc = "Negate each element of the vector."] @@ -1998,6 +2044,8 @@ pub trait Simd: a: i8x64, b: i8x64, ) -> i8x64; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2082,6 +2130,8 @@ pub trait Simd: a: u8x64, b: u8x64, ) -> u8x64; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2205,6 +2255,12 @@ pub trait Simd: a: i16x32, b: i16x32, ) -> i16x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2289,6 +2345,12 @@ pub trait Simd: a: u16x32, b: u16x32, ) -> u16x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2416,6 +2478,12 @@ pub trait Simd: a: i32x16, b: i32x16, ) -> i32x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2502,6 +2570,12 @@ pub trait Simd: a: u32x16, b: u32x16, ) -> u32x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2627,6 +2701,8 @@ pub trait Simd: a: f64x8, b: f64x8, ) -> f64x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; #[doc = "Compute the absolute value of each element."] fn abs_f64x8(self, a: f64x8) -> f64x8; #[doc = "Negate each element of the vector."] @@ -2808,6 +2884,8 @@ pub trait SimdBase: type Mask: SimdMask::Mask>; #[doc = r" A 128-bit SIMD vector of the same scalar type."] type Block: SimdBase; + #[doc = r" A same-width byte vector used to dynamically index bytes in this vector."] + type Indices: SimdBase; #[doc = r" The array type that this vector type corresponds to, which will"] #[doc = r" always be `[Self::Element; Self::N]`. It has the same layout as"] #[doc = r" this vector type, but likely has a lower alignment."] @@ -2837,6 +2915,8 @@ pub trait SimdBase: fn slide(self, rhs: impl SimdInto) -> Self; #[doc = "Like `slide`, but operates independently on each 128-bit block."] fn slide_within_blocks(self, rhs: impl SimdInto) -> Self; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; } #[doc = r" Functionality implemented by floating-point SIMD vectors."] pub trait SimdFloat: diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index c05fa1b73..2082b4869 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -86,6 +86,7 @@ impl SimdBase for f32x4 { const N: usize = 4; type Mask = mask32x4; type Block = f32x4; + type Indices = u8x16; type Array = [f32; 4]; #[inline(always)] fn witness(&self) -> S { @@ -130,6 +131,11 @@ impl SimdBase for f32x4 { self.simd .slide_within_blocks_f32x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x4 { #[inline(always)] @@ -339,6 +345,7 @@ impl SimdBase for i8x16 { const N: usize = 16; type Mask = mask8x16; type Block = i8x16; + type Indices = u8x16; type Array = [i8; 16]; #[inline(always)] fn witness(&self) -> S { @@ -383,6 +390,11 @@ impl SimdBase for i8x16 { self.simd .slide_within_blocks_i8x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x16 { #[inline(always)] @@ -524,6 +536,7 @@ impl SimdBase for u8x16 { const N: usize = 16; type Mask = mask8x16; type Block = u8x16; + type Indices = u8x16; type Array = [u8; 16]; #[inline(always)] fn witness(&self) -> S { @@ -568,6 +581,11 @@ impl SimdBase for u8x16 { self.simd .slide_within_blocks_u8x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x16 { #[inline(always)] @@ -800,6 +818,7 @@ impl SimdBase for i16x8 { const N: usize = 8; type Mask = mask16x8; type Block = i16x8; + type Indices = u8x16; type Array = [i16; 8]; #[inline(always)] fn witness(&self) -> S { @@ -844,6 +863,11 @@ impl SimdBase for i16x8 { self.simd .slide_within_blocks_i16x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x8 { #[inline(always)] @@ -985,6 +1009,7 @@ impl SimdBase for u16x8 { const N: usize = 8; type Mask = mask16x8; type Block = u16x8; + type Indices = u8x16; type Array = [u16; 8]; #[inline(always)] fn witness(&self) -> S { @@ -1029,6 +1054,11 @@ impl SimdBase for u16x8 { self.simd .slide_within_blocks_u16x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x8 { #[inline(always)] @@ -1261,6 +1291,7 @@ impl SimdBase for i32x4 { const N: usize = 4; type Mask = mask32x4; type Block = i32x4; + type Indices = u8x16; type Array = [i32; 4]; #[inline(always)] fn witness(&self) -> S { @@ -1305,6 +1336,11 @@ impl SimdBase for i32x4 { self.simd .slide_within_blocks_i32x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x4 { #[inline(always)] @@ -1458,6 +1494,7 @@ impl SimdBase for u32x4 { const N: usize = 4; type Mask = mask32x4; type Block = u32x4; + type Indices = u8x16; type Array = [u32; 4]; #[inline(always)] fn witness(&self) -> S { @@ -1502,6 +1539,11 @@ impl SimdBase for u32x4 { self.simd .slide_within_blocks_u32x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x4 { #[inline(always)] @@ -1746,6 +1788,7 @@ impl SimdBase for f64x2 { const N: usize = 2; type Mask = mask64x2; type Block = f64x2; + type Indices = u8x16; type Array = [f64; 2]; #[inline(always)] fn witness(&self) -> S { @@ -1790,6 +1833,11 @@ impl SimdBase for f64x2 { self.simd .slide_within_blocks_f64x2::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x2 { #[inline(always)] @@ -2076,6 +2124,7 @@ impl SimdBase for f32x8 { const N: usize = 8; type Mask = mask32x8; type Block = f32x4; + type Indices = u8x32; type Array = [f32; 8]; #[inline(always)] fn witness(&self) -> S { @@ -2120,6 +2169,11 @@ impl SimdBase for f32x8 { self.simd .slide_within_blocks_f32x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x8 { #[inline(always)] @@ -2336,6 +2390,7 @@ impl SimdBase for i8x32 { const N: usize = 32; type Mask = mask8x32; type Block = i8x16; + type Indices = u8x32; type Array = [i8; 32]; #[inline(always)] fn witness(&self) -> S { @@ -2380,6 +2435,11 @@ impl SimdBase for i8x32 { self.simd .slide_within_blocks_i8x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x32 { #[inline(always)] @@ -2528,6 +2588,7 @@ impl SimdBase for u8x32 { const N: usize = 32; type Mask = mask8x32; type Block = u8x16; + type Indices = u8x32; type Array = [u8; 32]; #[inline(always)] fn witness(&self) -> S { @@ -2572,6 +2633,11 @@ impl SimdBase for u8x32 { self.simd .slide_within_blocks_u8x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x32 { #[inline(always)] @@ -2816,6 +2882,7 @@ impl SimdBase for i16x16 { const N: usize = 16; type Mask = mask16x16; type Block = i16x8; + type Indices = u8x32; type Array = [i16; 16]; #[inline(always)] fn witness(&self) -> S { @@ -2860,6 +2927,11 @@ impl SimdBase for i16x16 { self.simd .slide_within_blocks_i16x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x16 { #[inline(always)] @@ -3014,6 +3086,7 @@ impl SimdBase for u16x16 { const N: usize = 16; type Mask = mask16x16; type Block = u16x8; + type Indices = u8x32; type Array = [u16; 16]; #[inline(always)] fn witness(&self) -> S { @@ -3058,6 +3131,11 @@ impl SimdBase for u16x16 { self.simd .slide_within_blocks_u16x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x16 { #[inline(always)] @@ -3298,6 +3376,7 @@ impl SimdBase for i32x8 { const N: usize = 8; type Mask = mask32x8; type Block = i32x4; + type Indices = u8x32; type Array = [i32; 8]; #[inline(always)] fn witness(&self) -> S { @@ -3342,6 +3421,11 @@ impl SimdBase for i32x8 { self.simd .slide_within_blocks_i32x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x8 { #[inline(always)] @@ -3502,6 +3586,7 @@ impl SimdBase for u32x8 { const N: usize = 8; type Mask = mask32x8; type Block = u32x4; + type Indices = u8x32; type Array = [u32; 8]; #[inline(always)] fn witness(&self) -> S { @@ -3546,6 +3631,11 @@ impl SimdBase for u32x8 { self.simd .slide_within_blocks_u32x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x8 { #[inline(always)] @@ -3797,6 +3887,7 @@ impl SimdBase for f64x4 { const N: usize = 4; type Mask = mask64x4; type Block = f64x2; + type Indices = u8x32; type Array = [f64; 4]; #[inline(always)] fn witness(&self) -> S { @@ -3841,6 +3932,11 @@ impl SimdBase for f64x4 { self.simd .slide_within_blocks_f64x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x4 { #[inline(always)] @@ -4139,6 +4235,7 @@ impl SimdBase for f32x16 { const N: usize = 16; type Mask = mask32x16; type Block = f32x4; + type Indices = u8x64; type Array = [f32; 16]; #[inline(always)] fn witness(&self) -> S { @@ -4184,6 +4281,11 @@ impl SimdBase for f32x16 { self.simd .slide_within_blocks_f32x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x16 { #[inline(always)] @@ -4394,6 +4496,7 @@ impl SimdBase for i8x64 { const N: usize = 64; type Mask = mask8x64; type Block = i8x16; + type Indices = u8x64; type Array = [i8; 64]; #[inline(always)] fn witness(&self) -> S { @@ -4439,6 +4542,11 @@ impl SimdBase for i8x64 { self.simd .slide_within_blocks_i8x64::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x64 { #[inline(always)] @@ -4580,6 +4688,7 @@ impl SimdBase for u8x64 { const N: usize = 64; type Mask = mask8x64; type Block = u8x16; + type Indices = u8x64; type Array = [u8; 64]; #[inline(always)] fn witness(&self) -> S { @@ -4625,6 +4734,11 @@ impl SimdBase for u8x64 { self.simd .slide_within_blocks_u8x64::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x64 { #[inline(always)] @@ -4862,6 +4976,7 @@ impl SimdBase for i16x32 { const N: usize = 32; type Mask = mask16x32; type Block = i16x8; + type Indices = u8x64; type Array = [i16; 32]; #[inline(always)] fn witness(&self) -> S { @@ -4907,6 +5022,11 @@ impl SimdBase for i16x32 { self.simd .slide_within_blocks_i16x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x32 { #[inline(always)] @@ -5054,6 +5174,7 @@ impl SimdBase for u16x32 { const N: usize = 32; type Mask = mask16x32; type Block = u16x8; + type Indices = u8x64; type Array = [u16; 32]; #[inline(always)] fn witness(&self) -> S { @@ -5099,6 +5220,11 @@ impl SimdBase for u16x32 { self.simd .slide_within_blocks_u16x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x32 { #[inline(always)] @@ -5337,6 +5463,7 @@ impl SimdBase for i32x16 { const N: usize = 16; type Mask = mask32x16; type Block = i32x4; + type Indices = u8x64; type Array = [i32; 16]; #[inline(always)] fn witness(&self) -> S { @@ -5382,6 +5509,11 @@ impl SimdBase for i32x16 { self.simd .slide_within_blocks_i32x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x16 { #[inline(always)] @@ -5541,6 +5673,7 @@ impl SimdBase for u32x16 { const N: usize = 16; type Mask = mask32x16; type Block = u32x4; + type Indices = u8x64; type Array = [u32; 16]; #[inline(always)] fn witness(&self) -> S { @@ -5586,6 +5719,11 @@ impl SimdBase for u32x16 { self.simd .slide_within_blocks_u32x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x16 { #[inline(always)] @@ -5831,6 +5969,7 @@ impl SimdBase for f64x8 { const N: usize = 8; type Mask = mask64x8; type Block = f64x2; + type Indices = u8x64; type Array = [f64; 8]; #[inline(always)] fn witness(&self) -> S { @@ -5876,6 +6015,11 @@ impl SimdBase for f64x8 { self.simd .slide_within_blocks_f64x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x8 { #[inline(always)] diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index abbac0c52..5d0929e33 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -201,6 +201,20 @@ impl Simd for Sse4_2 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x4, indices: u8x16) -> f32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -708,6 +722,20 @@ impl Simd for Sse4_2 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x16, indices: u8x16) -> i8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1073,6 +1101,20 @@ impl Simd for Sse4_2 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x16, indices: u8x16) -> u8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1611,6 +1653,20 @@ impl Simd for Sse4_2 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x8, indices: u8x16) -> i16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -1957,6 +2013,20 @@ impl Simd for Sse4_2 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x8, indices: u8x16) -> u16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2471,6 +2541,20 @@ impl Simd for Sse4_2 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x4, indices: u8x16) -> i32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -2825,6 +2909,20 @@ impl Simd for Sse4_2 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x4, indices: u8x16) -> u32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3343,6 +3441,20 @@ impl Simd for Sse4_2 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x2, indices: u8x16) -> f64x2 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -3906,6 +4018,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4277,6 +4398,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4555,6 +4685,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -4952,6 +5091,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5230,6 +5382,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -5648,6 +5813,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -5931,6 +6105,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6325,6 +6508,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -6773,6 +6965,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -7212,6 +7417,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -7483,6 +7697,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -7987,6 +8210,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -8267,6 +8503,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -8761,6 +9010,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -9037,6 +9299,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -9492,6 +9767,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 09a2c0048..8161c57c8 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -157,6 +157,14 @@ impl Simd for WasmSimd128 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_f32x4(a).val.0, indices.into()); + self.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { f32x4_abs(a.into()).simd_into(self) } @@ -451,6 +459,14 @@ impl Simd for WasmSimd128 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let result = u8x16_swizzle(self.cvt_to_bytes_i8x16(a).val.0, indices.into()); + self.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { i8x16_add(a.into(), b.into()).simd_into(self) } @@ -664,6 +680,14 @@ impl Simd for WasmSimd128 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let result = u8x16_swizzle(self.cvt_to_bytes_u8x16(a).val.0, indices.into()); + self.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { u8x16_add(a.into(), b.into()).simd_into(self) } @@ -974,6 +998,14 @@ impl Simd for WasmSimd128 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let result = u8x16_swizzle(self.cvt_to_bytes_i16x8(a).val.0, indices.into()); + self.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { i16x8_add(a.into(), b.into()).simd_into(self) } @@ -1171,6 +1203,14 @@ impl Simd for WasmSimd128 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let result = u8x16_swizzle(self.cvt_to_bytes_u16x8(a).val.0, indices.into()); + self.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { u16x8_add(a.into(), b.into()).simd_into(self) } @@ -1461,6 +1501,14 @@ impl Simd for WasmSimd128 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_i32x4(a).val.0, indices.into()); + self.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { i32x4_add(a.into(), b.into()).simd_into(self) } @@ -1662,6 +1710,14 @@ impl Simd for WasmSimd128 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_u32x4(a).val.0, indices.into()); + self.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { u32x4_add(a.into(), b.into()).simd_into(self) } @@ -1952,6 +2008,14 @@ impl Simd for WasmSimd128 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let result = u8x16_swizzle(self.cvt_to_bytes_f64x2(a).val.0, indices.into()); + self.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { f64x2_abs(a.into()).simd_into(self) } @@ -2307,6 +2371,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -2677,6 +2750,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -2954,6 +3036,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -3350,6 +3441,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -3627,6 +3731,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -4032,6 +4149,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -4314,6 +4440,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -4707,6 +4842,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -5154,6 +5298,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -5563,6 +5720,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -5833,6 +5999,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -6280,6 +6455,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -6559,6 +6747,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -7004,6 +7205,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -7279,6 +7493,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -7704,6 +7931,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 96b710c02..8e041e149 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -69,6 +69,20 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } } + OpSig::SwizzleDynWithinBlocks => { + let indices_ty = ty.indices_ty(); + let split_indices = generic_op_name("split", &indices_ty); + quote! { + #method_sig { + let (a0, a1) = self.#split(a); + let (indices0, indices1) = self.#split_indices(indices); + self.#combine( + self.#do_half(a0, indices0), + self.#do_half(a1, indices1), + ) + } + } + } OpSig::Ternary => { quote! { #method_sig { diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 92099258a..ff8509b60 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -392,6 +392,38 @@ impl Level for Fallback { } } } + OpSig::SwizzleDynWithinBlocks => { + assert_eq!( + vec_ty.n_bits(), + self.native_width(), + "wide swizzles should use the generic split implementation" + ); + let bytes_ty = vec_ty.indices_ty(); + let bytes_rust = bytes_ty.rust(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + let byte_count = bytes_ty.len; + let items = make_list( + (0..byte_count) + .map(|idx| { + quote! { + { + let index = indices[#idx] as usize; + bytes[index % #byte_count] + } + } + }) + .collect::>(), + ); + + quote! { + #method_sig { + let bytes = self.#to_bytes(a); + let result: #bytes_rust = #items.simd_into(self); + self.#from_bytes(result) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 2cdf737f4..84bb18193 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -424,6 +424,26 @@ impl Level for Neon { } } } + OpSig::SwizzleDynWithinBlocks => { + assert_eq!( + vec_ty.n_bits(), + self.native_width(), + "wide swizzles should use the generic split implementation" + ); + + let bytes_ty = vec_ty.indices_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + self.kernel_method(op, vec_ty, |token| { + quote! { + let result = vqtbl1q_u8(#token.#to_bytes(a).val.0, indices.into()); + #token.#from_bytes(#bytes { val: #wrapper(result), simd: #token }) + } + }) + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_simd_trait.rs b/fearless_simd_gen/src/mk_simd_trait.rs index 99fb91a76..1c6469b93 100644 --- a/fearless_simd_gen/src/mk_simd_trait.rs +++ b/fearless_simd_gen/src/mk_simd_trait.rs @@ -168,6 +168,8 @@ fn mk_simd_base() -> TokenStream { type Mask: SimdMask::Mask>; /// A 128-bit SIMD vector of the same scalar type. type Block: SimdBase; + /// A same-width byte vector used to dynamically index bytes in this vector. + type Indices: SimdBase; /// The array type that this vector type corresponds to, which will /// always be `[Self::Element; Self::N]`. It has the same layout as /// this vector type, but likely has a lower alignment. diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index b6f2aafce..4f688e597 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -399,6 +399,7 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { } let mask_ty = ty.mask_ty().rust(); let block_ty = ty.block_ty().rust(); + let indices_ty = ty.indices_ty().rust(); let block_splat_body = match ty.n_bits() { 128 => quote! { block @@ -429,12 +430,14 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { let as_array_mut_op = generic_op_name("as_array_mut", ty); let slide_op = generic_op_name("slide", ty); let slide_blockwise_op = generic_op_name("slide_within_blocks", ty); + let swizzle_dyn_within_blocks_op = generic_op_name("swizzle_dyn_within_blocks", ty); quote! { impl SimdBase for #name { type Element = #scalar; const N: usize = #len; type Mask = #mask_ty; type Block = #block_ty; + type Indices = #indices_ty; type Array = [#scalar; #len]; #[inline(always)] @@ -486,6 +489,11 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { self.simd.#slide_blockwise_op::(self, rhs.simd_into(self.simd)) } + + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd.#swizzle_dyn_within_blocks_op(self, indices.simd_into(self.simd)) + } } impl crate::#vec_trait_id for #name { #( #methods )* diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index f62014bdd..43d7388d1 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -483,6 +483,26 @@ impl Level for WasmSimd128 { } } } + OpSig::SwizzleDynWithinBlocks => { + assert_eq!( + vec_ty.n_bits(), + self.native_width(), + "wide swizzles should use the generic split implementation" + ); + + let bytes_ty = vec_ty.indices_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + quote! { + #method_sig { + let result = u8x16_swizzle(self.#to_bytes(a).val.0, indices.into()); + self.#from_bytes(#bytes { val: #wrapper(result), simd: self }) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 724c12ef5..4d65b8171 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -306,6 +306,7 @@ impl Level for X86 { OpSig::Zip { select_low } => self.handle_zip(op, vec_ty, select_low), OpSig::Unzip { select_even } => self.handle_unzip(op, vec_ty, select_even), OpSig::Slide { granularity } => self.handle_slide(method_sig, vec_ty, granularity), + OpSig::SwizzleDynWithinBlocks => self.handle_swizzle_dyn_within_blocks(op, vec_ty), OpSig::Cvt { target_ty, scalar_bits, @@ -2463,6 +2464,43 @@ impl X86 { } } + pub(crate) fn handle_swizzle_dyn_within_blocks(&self, op: Op, vec_ty: &VecType) -> TokenStream { + let bytes_ty = vec_ty.indices_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + self.kernel_method(op, vec_ty, |token| { + let body = if *self == Self::Avx512 { + match vec_ty.n_bits() { + 128 => quote! { + let bytes = #token.#to_bytes(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + }, + 256 => quote! { + let bytes = #token.#to_bytes(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + }, + 512 => quote! { + let result = _mm512_shuffle_epi8(#token.#to_bytes(a).val.0, indices.into()); + }, + _ => unreachable!(), + } + } else { + let shuffle = simple_sign_unaware_intrinsic("shuffle", &bytes_ty); + quote! { + let result = #shuffle(#token.#to_bytes(a).val.0, indices.into()); + } + }; + + quote! { + #body + #token.#from_bytes(#bytes { val: #wrapper(result), simd: #token }) + } + }) + } + pub(crate) fn handle_cvt( &self, op: Op, diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 17495b41b..75d4de963 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -82,6 +82,9 @@ pub(crate) enum OpSig { Deinterleave, /// Takes two arguments of a vector type, plus a const generic shift amount, and returns that same vector type. Slide { granularity: SlideGranularity }, + /// Takes a vector and a same-width byte-index vector, and returns the original vector type with its bytes + /// dynamically swizzled within each 128-bit block. + SwizzleDynWithinBlocks, /// Takes a single argument of the source vector type, and returns a vector type of the target scalar type and the /// same length. Cvt { @@ -304,6 +307,10 @@ impl Op { (vec![vec.clone(), vec.clone()], quote! { (#vec, #vec) }) } OpSig::Slide { .. } => (vec![vec.clone(), vec.clone()], vec), + OpSig::SwizzleDynWithinBlocks => { + let indices_ty = vec_ty.indices_ty().rust(); + (vec![vec.clone(), quote! { #indices_ty<#simd_ty> }], vec) + } OpSig::Cvt { target_ty, scalar_bits, @@ -421,6 +428,11 @@ impl Op { let arg1 = &arg_names[1]; quote! { (#arg0, #arg1: impl SimdInto) -> Self } } + OpSig::SwizzleDynWithinBlocks => { + let arg0 = &arg_names[0]; + let arg1 = &arg_names[1]; + quote! { (#arg0, #arg1: impl SimdInto) -> Self } + } OpSig::Compare => { let arg0 = &arg_names[0]; let arg1 = &arg_names[1]; @@ -587,6 +599,14 @@ const BASE_OPS: &[Op] = &[ }, "Like `slide`, but operates independently on each 128-bit block.", ), + Op::new( + "swizzle_dyn_within_blocks", + OpKind::BaseTraitMethod, + OpSig::SwizzleDynWithinBlocks, + "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\n\ + The `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\n\ + Out-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero.", + ), ]; const MASK_REPRESENTATION_OPS: &[Op] = &[ @@ -1611,6 +1631,7 @@ impl OpSig { | Self::AsArray { .. } | Self::FromBytes | Self::ToBytes => &["a"], + Self::SwizzleDynWithinBlocks => &["a", "indices"], Self::Binary | Self::Compare | Self::Combine { .. } @@ -1643,6 +1664,7 @@ impl OpSig { | Self::MaskReduce { .. } | Self::AsArray { .. } | Self::ToBytes => &["self"], + Self::SwizzleDynWithinBlocks => &["self", "indices"], Self::Binary | Self::Compare | Self::Zip { .. } @@ -1704,6 +1726,7 @@ impl OpSig { | Self::StoreArray | Self::FromBytes | Self::ToBytes + | Self::SwizzleDynWithinBlocks | Self::Slide { .. } => return None, }; Some(args) diff --git a/fearless_simd_gen/src/types.rs b/fearless_simd_gen/src/types.rs index 670d6a5cb..a1a7d1996 100644 --- a/fearless_simd_gen/src/types.rs +++ b/fearless_simd_gen/src/types.rs @@ -157,6 +157,10 @@ impl VecType { Self::new(self.scalar, self.scalar_bits, 128 / self.scalar_bits) } + pub(crate) fn indices_ty(&self) -> Self { + Self::new(ScalarType::Unsigned, 8, self.n_bits() / 8) + } + pub(crate) fn split_operand(&self) -> Option { if self.n_bits() <= 128 { return None; diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index c250667c5..06e784e67 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -3794,6 +3794,397 @@ fn bitcast_native(simd: S) { ); } +fn expected_swizzle_within_blocks(bytes: [u8; N], indices: [u8; N]) -> [u8; N] { + assert_eq!(N % 16, 0); + core::array::from_fn(|i| { + let block_start = (i / 16) * 16; + bytes[block_start + usize::from(indices[i])] + }) +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f32x4(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: f32x4 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i8x16(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: i8x16 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u8x16(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: u8x16 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i16x8(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: i16x8 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u16x8(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: u16x8 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i32x4(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: i32x4 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u32x4(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: u32x4 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f64x2(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: f64x2 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: f32x8 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i8x32(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: i8x32 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u8x32(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: u8x32 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i16x16(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: i16x16 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u16x16(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: u16x16 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: i32x8 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: u32x8 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f64x4(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: f64x4 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f32x16(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: f32x16 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i8x64(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: i8x64 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u8x64(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: u8x64 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i16x32(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: i16x32 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u16x32(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: u16x32 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i32x16(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: i32x16 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u32x16(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: u32x16 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f64x8(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); + let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: f64x8 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_generic_indices(simd: S) { + #[inline(always)] + fn do_swizzle>(value: V, indices: V::Indices) -> V { + value.swizzle_dyn_within_blocks(indices) + } + + let value = u8x16::from_fn(simd, |i| i as u8); + let indices = u8x16::from_fn(simd, |i| (15 - i) as u8); + let result = do_swizzle::>(value, indices); + + assert_eq!( + *result, + [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] + ); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_x86_pshufb_oob(simd: S) { + #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] + { + if !matches!( + simd.level(), + Level::Sse4_2(_) | Level::Avx2(_) | Level::Avx512(_) + ) { + return; + } + + let value = u8x16::from_fn(simd, |i| i as u8); + let indices = u8x16::simd_from( + simd, + [ + 16, 17, 31, 0x80, 0x8f, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, + ], + ); + let result = value.swizzle_dyn_within_blocks(indices); + + assert_eq!( + *result, + [0, 1, 15, 0, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5] + ); + } + + #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] + let _ = simd; +} + #[simd_test] fn wrapping_add_u32(simd: S) { assert_eq!( From 46e0931bcd81cbf7b47530ac361de16b8a2de7a7 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 2 Jul 2026 20:16:02 +0100 Subject: [PATCH 2/6] look what clippy made me do --- fearless_simd_tests/tests/harness/mod.rs | 111 ++++++++++++----------- 1 file changed, 60 insertions(+), 51 deletions(-) diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index 06e784e67..d8a4d5fe8 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -3802,10 +3802,19 @@ fn expected_swizzle_within_blocks(bytes: [u8; N], indices: [u8; }) } +#[allow(clippy::cast_possible_truncation)] // truncation is deliberate +fn swizzle_test_byte(i: usize) -> u8 { + (i * 37 + 11) as u8 +} + +fn swizzle_test_index(i: usize) -> u8 { + u8::try_from(15 - (i % 16)).unwrap() +} + #[simd_test] fn swizzle_dyn_within_blocks_f32x4(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3818,8 +3827,8 @@ fn swizzle_dyn_within_blocks_f32x4(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i8x16(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3832,8 +3841,8 @@ fn swizzle_dyn_within_blocks_i8x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u8x16(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3846,8 +3855,8 @@ fn swizzle_dyn_within_blocks_u8x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i16x8(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3860,8 +3869,8 @@ fn swizzle_dyn_within_blocks_i16x8(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u16x8(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3874,8 +3883,8 @@ fn swizzle_dyn_within_blocks_u16x8(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i32x4(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3888,8 +3897,8 @@ fn swizzle_dyn_within_blocks_i32x4(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u32x4(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3902,8 +3911,8 @@ fn swizzle_dyn_within_blocks_u32x4(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_f64x2(simd: S) { - let bytes: [u8; 16] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 16] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x16::simd_from(simd, bytes); @@ -3916,8 +3925,8 @@ fn swizzle_dyn_within_blocks_f64x2(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_f32x8(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -3930,8 +3939,8 @@ fn swizzle_dyn_within_blocks_f32x8(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i8x32(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -3944,8 +3953,8 @@ fn swizzle_dyn_within_blocks_i8x32(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u8x32(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -3958,8 +3967,8 @@ fn swizzle_dyn_within_blocks_u8x32(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i16x16(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -3972,8 +3981,8 @@ fn swizzle_dyn_within_blocks_i16x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u16x16(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -3986,8 +3995,8 @@ fn swizzle_dyn_within_blocks_u16x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i32x8(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -4000,8 +4009,8 @@ fn swizzle_dyn_within_blocks_i32x8(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u32x8(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -4014,8 +4023,8 @@ fn swizzle_dyn_within_blocks_u32x8(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_f64x4(simd: S) { - let bytes: [u8; 32] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 32] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x32::simd_from(simd, bytes); @@ -4028,8 +4037,8 @@ fn swizzle_dyn_within_blocks_f64x4(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_f32x16(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4042,8 +4051,8 @@ fn swizzle_dyn_within_blocks_f32x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i8x64(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4056,8 +4065,8 @@ fn swizzle_dyn_within_blocks_i8x64(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u8x64(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4070,8 +4079,8 @@ fn swizzle_dyn_within_blocks_u8x64(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i16x32(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4084,8 +4093,8 @@ fn swizzle_dyn_within_blocks_i16x32(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u16x32(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4098,8 +4107,8 @@ fn swizzle_dyn_within_blocks_u16x32(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_i32x16(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4112,8 +4121,8 @@ fn swizzle_dyn_within_blocks_i32x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_u32x16(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4126,8 +4135,8 @@ fn swizzle_dyn_within_blocks_u32x16(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_f64x8(simd: S) { - let bytes: [u8; 64] = core::array::from_fn(|i| ((i * 37 + 11) & 0xff) as u8); - let indices: [u8; 64] = core::array::from_fn(|i| (15 - (i % 16)) as u8); + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); let expected = expected_swizzle_within_blocks(bytes, indices); let byte_vec = u8x64::simd_from(simd, bytes); @@ -4145,8 +4154,8 @@ fn swizzle_dyn_within_blocks_generic_indices(simd: S) { value.swizzle_dyn_within_blocks(indices) } - let value = u8x16::from_fn(simd, |i| i as u8); - let indices = u8x16::from_fn(simd, |i| (15 - i) as u8); + let value = u8x16::from_fn(simd, |i| u8::try_from(i).unwrap()); + let indices = u8x16::from_fn(simd, |i| u8::try_from(15 - i).unwrap()); let result = do_swizzle::>(value, indices); assert_eq!( @@ -4166,7 +4175,7 @@ fn swizzle_dyn_within_blocks_x86_pshufb_oob(simd: S) { return; } - let value = u8x16::from_fn(simd, |i| i as u8); + let value = u8x16::from_fn(simd, |i| u8::try_from(i).unwrap()); let indices = u8x16::simd_from( simd, [ From fd75b7f53daf37543b492c2f6ba155a72f8e2587 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Thu, 2 Jul 2026 20:19:30 +0100 Subject: [PATCH 3/6] placate Clippy harder --- fearless_simd_tests/tests/harness/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index d8a4d5fe8..da1960e08 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -3802,7 +3802,7 @@ fn expected_swizzle_within_blocks(bytes: [u8; N], indices: [u8; }) } -#[allow(clippy::cast_possible_truncation)] // truncation is deliberate +#[expect(clippy::cast_possible_truncation, reason = "truncation is deliberate")] fn swizzle_test_byte(i: usize) -> u8 { (i * 37 + 11) as u8 } From 28c2d9a3ab9a08c041ed40c3a4343215c7b05ff0 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Fri, 3 Jul 2026 09:22:55 +0100 Subject: [PATCH 4/6] Empty commit to re-run stuck CI From 849822ae2dd617dbe8f26df44e14ca8e54ff45a6 Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 5 Jul 2026 19:38:25 +0100 Subject: [PATCH 5/6] Do not document and test target-specific behavior of swizzle_within_blocks --- fearless_simd/src/generated/simd_trait.rs | 50 +++++++++++------------ fearless_simd_gen/src/ops.rs | 2 +- fearless_simd_tests/tests/harness/mod.rs | 36 ++++------------ 3 files changed, 35 insertions(+), 53 deletions(-) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index fd6d09314..e1daf16a6 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -144,7 +144,7 @@ pub trait Simd: a: f32x4, b: f32x4, ) -> f32x4; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; #[doc = "Compute the absolute value of each element."] fn abs_f32x4(self, a: f32x4) -> f32x4; @@ -254,7 +254,7 @@ pub trait Simd: a: i8x16, b: i8x16, ) -> i8x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16; @@ -340,7 +340,7 @@ pub trait Simd: a: u8x16, b: u8x16, ) -> u8x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16; @@ -463,7 +463,7 @@ pub trait Simd: a: i16x8, b: i16x8, ) -> i16x8; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8; @@ -549,7 +549,7 @@ pub trait Simd: a: u16x8, b: u16x8, ) -> u16x8; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8; @@ -672,7 +672,7 @@ pub trait Simd: a: i32x4, b: i32x4, ) -> i32x4; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4; @@ -760,7 +760,7 @@ pub trait Simd: a: u32x4, b: u32x4, ) -> u32x4; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4; @@ -883,7 +883,7 @@ pub trait Simd: a: f64x2, b: f64x2, ) -> f64x2; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; #[doc = "Compute the absolute value of each element."] fn abs_f64x2(self, a: f64x2) -> f64x2; @@ -1018,7 +1018,7 @@ pub trait Simd: a: f32x8, b: f32x8, ) -> f32x8; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; #[doc = "Compute the absolute value of each element."] fn abs_f32x8(self, a: f32x8) -> f32x8; @@ -1130,7 +1130,7 @@ pub trait Simd: a: i8x32, b: i8x32, ) -> i8x32; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32; @@ -1218,7 +1218,7 @@ pub trait Simd: a: u8x32, b: u8x32, ) -> u8x32; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32; @@ -1345,7 +1345,7 @@ pub trait Simd: a: i16x16, b: i16x16, ) -> i16x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i16x16( self, a: i16x16, @@ -1437,7 +1437,7 @@ pub trait Simd: a: u16x16, b: u16x16, ) -> u16x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u16x16( self, a: u16x16, @@ -1570,7 +1570,7 @@ pub trait Simd: a: i32x8, b: i32x8, ) -> i32x8; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8; @@ -1660,7 +1660,7 @@ pub trait Simd: a: u32x8, b: u32x8, ) -> u32x8; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8; @@ -1787,7 +1787,7 @@ pub trait Simd: a: f64x4, b: f64x4, ) -> f64x4; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; #[doc = "Compute the absolute value of each element."] fn abs_f64x4(self, a: f64x4) -> f64x4; @@ -1926,7 +1926,7 @@ pub trait Simd: a: f32x16, b: f32x16, ) -> f32x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f32x16( self, a: f32x16, @@ -2044,7 +2044,7 @@ pub trait Simd: a: i8x64, b: i8x64, ) -> i8x64; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64; @@ -2130,7 +2130,7 @@ pub trait Simd: a: u8x64, b: u8x64, ) -> u8x64; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64; @@ -2255,7 +2255,7 @@ pub trait Simd: a: i16x32, b: i16x32, ) -> i16x32; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i16x32( self, a: i16x32, @@ -2345,7 +2345,7 @@ pub trait Simd: a: u16x32, b: u16x32, ) -> u16x32; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u16x32( self, a: u16x32, @@ -2478,7 +2478,7 @@ pub trait Simd: a: i32x16, b: i32x16, ) -> i32x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_i32x16( self, a: i32x16, @@ -2570,7 +2570,7 @@ pub trait Simd: a: u32x16, b: u32x16, ) -> u32x16; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_u32x16( self, a: u32x16, @@ -2701,7 +2701,7 @@ pub trait Simd: a: f64x8, b: f64x8, ) -> f64x8; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; #[doc = "Compute the absolute value of each element."] fn abs_f64x8(self, a: f64x8) -> f64x8; @@ -2915,7 +2915,7 @@ pub trait SimdBase: fn slide(self, rhs: impl SimdInto) -> Self; #[doc = "Like `slide`, but operates independently on each 128-bit block."] fn slide_within_blocks(self, rhs: impl SimdInto) -> Self; - #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero."] + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; } #[doc = r" Functionality implemented by floating-point SIMD vectors."] diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 75d4de963..0972ae6f2 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -605,7 +605,7 @@ const BASE_OPS: &[Op] = &[ OpSig::SwizzleDynWithinBlocks, "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\n\ The `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\n\ - Out-of-range index behavior is target-native: on x86 this follows `pshufb`, while NEON and WASM table swizzles produce zero.", + Out-of-range index behavior varies by platform.", ), ]; diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index da1960e08..d0f21b72d 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -4165,33 +4165,15 @@ fn swizzle_dyn_within_blocks_generic_indices(simd: S) { } #[simd_test] -fn swizzle_dyn_within_blocks_x86_pshufb_oob(simd: S) { - #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] - { - if !matches!( - simd.level(), - Level::Sse4_2(_) | Level::Avx2(_) | Level::Avx512(_) - ) { - return; - } - - let value = u8x16::from_fn(simd, |i| u8::try_from(i).unwrap()); - let indices = u8x16::simd_from( - simd, - [ - 16, 17, 31, 0x80, 0x8f, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, - ], - ); - let result = value.swizzle_dyn_within_blocks(indices); - - assert_eq!( - *result, - [0, 1, 15, 0, 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5] - ); - } - - #[cfg(not(any(target_arch = "x86", target_arch = "x86_64")))] - let _ = simd; +fn swizzle_dyn_within_blocks_oob_does_not_panic(simd: S) { + let value = u8x16::from_fn(simd, |i| u8::try_from(i).unwrap()); + let indices = u8x16::simd_from( + simd, + [ + 16, 17, 31, 0x80, 0x8f, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, + ], + ); + let _ = value.swizzle_dyn_within_blocks(indices); } #[simd_test] From ce46dce1f460f1c258d9940b974906d73c27076e Mon Sep 17 00:00:00 2001 From: "Sergey \"Shnatsel\" Davidoff" Date: Sun, 5 Jul 2026 19:50:51 +0100 Subject: [PATCH 6/6] Get rid of Indices type and just use Bytes that I didn't realize existed --- fearless_simd/src/generated/simd_trait.rs | 4 +- fearless_simd/src/generated/simd_types.rs | 72 ++++++++--------------- fearless_simd_gen/src/generic.rs | 4 +- fearless_simd_gen/src/mk_fallback.rs | 2 +- fearless_simd_gen/src/mk_neon.rs | 2 +- fearless_simd_gen/src/mk_simd_trait.rs | 2 - fearless_simd_gen/src/mk_simd_types.rs | 6 +- fearless_simd_gen/src/mk_wasm.rs | 2 +- fearless_simd_gen/src/mk_x86.rs | 2 +- fearless_simd_gen/src/ops.rs | 6 +- fearless_simd_gen/src/types.rs | 2 +- fearless_simd_tests/tests/harness/mod.rs | 2 +- 12 files changed, 38 insertions(+), 68 deletions(-) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index e1daf16a6..324b8457f 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -2884,8 +2884,6 @@ pub trait SimdBase: type Mask: SimdMask::Mask>; #[doc = r" A 128-bit SIMD vector of the same scalar type."] type Block: SimdBase; - #[doc = r" A same-width byte vector used to dynamically index bytes in this vector."] - type Indices: SimdBase; #[doc = r" The array type that this vector type corresponds to, which will"] #[doc = r" always be `[Self::Element; Self::N]`. It has the same layout as"] #[doc = r" this vector type, but likely has a lower alignment."] @@ -2916,7 +2914,7 @@ pub trait SimdBase: #[doc = "Like `slide`, but operates independently on each 128-bit block."] fn slide_within_blocks(self, rhs: impl SimdInto) -> Self; #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; } #[doc = r" Functionality implemented by floating-point SIMD vectors."] pub trait SimdFloat: diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index 2082b4869..3964c796d 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -86,7 +86,6 @@ impl SimdBase for f32x4 { const N: usize = 4; type Mask = mask32x4; type Block = f32x4; - type Indices = u8x16; type Array = [f32; 4]; #[inline(always)] fn witness(&self) -> S { @@ -132,7 +131,7 @@ impl SimdBase for f32x4 { .slide_within_blocks_f32x4::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_f32x4(self, indices.simd_into(self.simd)) } @@ -345,7 +344,6 @@ impl SimdBase for i8x16 { const N: usize = 16; type Mask = mask8x16; type Block = i8x16; - type Indices = u8x16; type Array = [i8; 16]; #[inline(always)] fn witness(&self) -> S { @@ -391,7 +389,7 @@ impl SimdBase for i8x16 { .slide_within_blocks_i8x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i8x16(self, indices.simd_into(self.simd)) } @@ -536,7 +534,6 @@ impl SimdBase for u8x16 { const N: usize = 16; type Mask = mask8x16; type Block = u8x16; - type Indices = u8x16; type Array = [u8; 16]; #[inline(always)] fn witness(&self) -> S { @@ -582,7 +579,7 @@ impl SimdBase for u8x16 { .slide_within_blocks_u8x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u8x16(self, indices.simd_into(self.simd)) } @@ -818,7 +815,6 @@ impl SimdBase for i16x8 { const N: usize = 8; type Mask = mask16x8; type Block = i16x8; - type Indices = u8x16; type Array = [i16; 8]; #[inline(always)] fn witness(&self) -> S { @@ -864,7 +860,7 @@ impl SimdBase for i16x8 { .slide_within_blocks_i16x8::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i16x8(self, indices.simd_into(self.simd)) } @@ -1009,7 +1005,6 @@ impl SimdBase for u16x8 { const N: usize = 8; type Mask = mask16x8; type Block = u16x8; - type Indices = u8x16; type Array = [u16; 8]; #[inline(always)] fn witness(&self) -> S { @@ -1055,7 +1050,7 @@ impl SimdBase for u16x8 { .slide_within_blocks_u16x8::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u16x8(self, indices.simd_into(self.simd)) } @@ -1291,7 +1286,6 @@ impl SimdBase for i32x4 { const N: usize = 4; type Mask = mask32x4; type Block = i32x4; - type Indices = u8x16; type Array = [i32; 4]; #[inline(always)] fn witness(&self) -> S { @@ -1337,7 +1331,7 @@ impl SimdBase for i32x4 { .slide_within_blocks_i32x4::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i32x4(self, indices.simd_into(self.simd)) } @@ -1494,7 +1488,6 @@ impl SimdBase for u32x4 { const N: usize = 4; type Mask = mask32x4; type Block = u32x4; - type Indices = u8x16; type Array = [u32; 4]; #[inline(always)] fn witness(&self) -> S { @@ -1540,7 +1533,7 @@ impl SimdBase for u32x4 { .slide_within_blocks_u32x4::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u32x4(self, indices.simd_into(self.simd)) } @@ -1788,7 +1781,6 @@ impl SimdBase for f64x2 { const N: usize = 2; type Mask = mask64x2; type Block = f64x2; - type Indices = u8x16; type Array = [f64; 2]; #[inline(always)] fn witness(&self) -> S { @@ -1834,7 +1826,7 @@ impl SimdBase for f64x2 { .slide_within_blocks_f64x2::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_f64x2(self, indices.simd_into(self.simd)) } @@ -2124,7 +2116,6 @@ impl SimdBase for f32x8 { const N: usize = 8; type Mask = mask32x8; type Block = f32x4; - type Indices = u8x32; type Array = [f32; 8]; #[inline(always)] fn witness(&self) -> S { @@ -2170,7 +2161,7 @@ impl SimdBase for f32x8 { .slide_within_blocks_f32x8::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_f32x8(self, indices.simd_into(self.simd)) } @@ -2390,7 +2381,6 @@ impl SimdBase for i8x32 { const N: usize = 32; type Mask = mask8x32; type Block = i8x16; - type Indices = u8x32; type Array = [i8; 32]; #[inline(always)] fn witness(&self) -> S { @@ -2436,7 +2426,7 @@ impl SimdBase for i8x32 { .slide_within_blocks_i8x32::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i8x32(self, indices.simd_into(self.simd)) } @@ -2588,7 +2578,6 @@ impl SimdBase for u8x32 { const N: usize = 32; type Mask = mask8x32; type Block = u8x16; - type Indices = u8x32; type Array = [u8; 32]; #[inline(always)] fn witness(&self) -> S { @@ -2634,7 +2623,7 @@ impl SimdBase for u8x32 { .slide_within_blocks_u8x32::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u8x32(self, indices.simd_into(self.simd)) } @@ -2882,7 +2871,6 @@ impl SimdBase for i16x16 { const N: usize = 16; type Mask = mask16x16; type Block = i16x8; - type Indices = u8x32; type Array = [i16; 16]; #[inline(always)] fn witness(&self) -> S { @@ -2928,7 +2916,7 @@ impl SimdBase for i16x16 { .slide_within_blocks_i16x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i16x16(self, indices.simd_into(self.simd)) } @@ -3086,7 +3074,6 @@ impl SimdBase for u16x16 { const N: usize = 16; type Mask = mask16x16; type Block = u16x8; - type Indices = u8x32; type Array = [u16; 16]; #[inline(always)] fn witness(&self) -> S { @@ -3132,7 +3119,7 @@ impl SimdBase for u16x16 { .slide_within_blocks_u16x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u16x16(self, indices.simd_into(self.simd)) } @@ -3376,7 +3363,6 @@ impl SimdBase for i32x8 { const N: usize = 8; type Mask = mask32x8; type Block = i32x4; - type Indices = u8x32; type Array = [i32; 8]; #[inline(always)] fn witness(&self) -> S { @@ -3422,7 +3408,7 @@ impl SimdBase for i32x8 { .slide_within_blocks_i32x8::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i32x8(self, indices.simd_into(self.simd)) } @@ -3586,7 +3572,6 @@ impl SimdBase for u32x8 { const N: usize = 8; type Mask = mask32x8; type Block = u32x4; - type Indices = u8x32; type Array = [u32; 8]; #[inline(always)] fn witness(&self) -> S { @@ -3632,7 +3617,7 @@ impl SimdBase for u32x8 { .slide_within_blocks_u32x8::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u32x8(self, indices.simd_into(self.simd)) } @@ -3887,7 +3872,6 @@ impl SimdBase for f64x4 { const N: usize = 4; type Mask = mask64x4; type Block = f64x2; - type Indices = u8x32; type Array = [f64; 4]; #[inline(always)] fn witness(&self) -> S { @@ -3933,7 +3917,7 @@ impl SimdBase for f64x4 { .slide_within_blocks_f64x4::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_f64x4(self, indices.simd_into(self.simd)) } @@ -4235,7 +4219,6 @@ impl SimdBase for f32x16 { const N: usize = 16; type Mask = mask32x16; type Block = f32x4; - type Indices = u8x64; type Array = [f32; 16]; #[inline(always)] fn witness(&self) -> S { @@ -4282,7 +4265,7 @@ impl SimdBase for f32x16 { .slide_within_blocks_f32x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_f32x16(self, indices.simd_into(self.simd)) } @@ -4496,7 +4479,6 @@ impl SimdBase for i8x64 { const N: usize = 64; type Mask = mask8x64; type Block = i8x16; - type Indices = u8x64; type Array = [i8; 64]; #[inline(always)] fn witness(&self) -> S { @@ -4543,7 +4525,7 @@ impl SimdBase for i8x64 { .slide_within_blocks_i8x64::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i8x64(self, indices.simd_into(self.simd)) } @@ -4688,7 +4670,6 @@ impl SimdBase for u8x64 { const N: usize = 64; type Mask = mask8x64; type Block = u8x16; - type Indices = u8x64; type Array = [u8; 64]; #[inline(always)] fn witness(&self) -> S { @@ -4735,7 +4716,7 @@ impl SimdBase for u8x64 { .slide_within_blocks_u8x64::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u8x64(self, indices.simd_into(self.simd)) } @@ -4976,7 +4957,6 @@ impl SimdBase for i16x32 { const N: usize = 32; type Mask = mask16x32; type Block = i16x8; - type Indices = u8x64; type Array = [i16; 32]; #[inline(always)] fn witness(&self) -> S { @@ -5023,7 +5003,7 @@ impl SimdBase for i16x32 { .slide_within_blocks_i16x32::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i16x32(self, indices.simd_into(self.simd)) } @@ -5174,7 +5154,6 @@ impl SimdBase for u16x32 { const N: usize = 32; type Mask = mask16x32; type Block = u16x8; - type Indices = u8x64; type Array = [u16; 32]; #[inline(always)] fn witness(&self) -> S { @@ -5221,7 +5200,7 @@ impl SimdBase for u16x32 { .slide_within_blocks_u16x32::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u16x32(self, indices.simd_into(self.simd)) } @@ -5463,7 +5442,6 @@ impl SimdBase for i32x16 { const N: usize = 16; type Mask = mask32x16; type Block = i32x4; - type Indices = u8x64; type Array = [i32; 16]; #[inline(always)] fn witness(&self) -> S { @@ -5510,7 +5488,7 @@ impl SimdBase for i32x16 { .slide_within_blocks_i32x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_i32x16(self, indices.simd_into(self.simd)) } @@ -5673,7 +5651,6 @@ impl SimdBase for u32x16 { const N: usize = 16; type Mask = mask32x16; type Block = u32x4; - type Indices = u8x64; type Array = [u32; 16]; #[inline(always)] fn witness(&self) -> S { @@ -5720,7 +5697,7 @@ impl SimdBase for u32x16 { .slide_within_blocks_u32x16::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_u32x16(self, indices.simd_into(self.simd)) } @@ -5969,7 +5946,6 @@ impl SimdBase for f64x8 { const N: usize = 8; type Mask = mask64x8; type Block = f64x2; - type Indices = u8x64; type Array = [f64; 8]; #[inline(always)] fn witness(&self) -> S { @@ -6016,7 +5992,7 @@ impl SimdBase for f64x8 { .slide_within_blocks_f64x8::(self, rhs.simd_into(self.simd)) } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd .swizzle_dyn_within_blocks_f64x8(self, indices.simd_into(self.simd)) } diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 8e041e149..485fcbbd0 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -70,8 +70,8 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } OpSig::SwizzleDynWithinBlocks => { - let indices_ty = ty.indices_ty(); - let split_indices = generic_op_name("split", &indices_ty); + let bytes_ty = ty.bytes_ty(); + let split_indices = generic_op_name("split", &bytes_ty); quote! { #method_sig { let (a0, a1) = self.#split(a); diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index ff8509b60..bf511662d 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -398,7 +398,7 @@ impl Level for Fallback { self.native_width(), "wide swizzles should use the generic split implementation" ); - let bytes_ty = vec_ty.indices_ty(); + let bytes_ty = vec_ty.bytes_ty(); let bytes_rust = bytes_ty.rust(); let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 84bb18193..0eb203ffc 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -431,7 +431,7 @@ impl Level for Neon { "wide swizzles should use the generic split implementation" ); - let bytes_ty = vec_ty.indices_ty(); + let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); diff --git a/fearless_simd_gen/src/mk_simd_trait.rs b/fearless_simd_gen/src/mk_simd_trait.rs index 1c6469b93..99fb91a76 100644 --- a/fearless_simd_gen/src/mk_simd_trait.rs +++ b/fearless_simd_gen/src/mk_simd_trait.rs @@ -168,8 +168,6 @@ fn mk_simd_base() -> TokenStream { type Mask: SimdMask::Mask>; /// A 128-bit SIMD vector of the same scalar type. type Block: SimdBase; - /// A same-width byte vector used to dynamically index bytes in this vector. - type Indices: SimdBase; /// The array type that this vector type corresponds to, which will /// always be `[Self::Element; Self::N]`. It has the same layout as /// this vector type, but likely has a lower alignment. diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index 4f688e597..8ca4d2bf9 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -32,7 +32,7 @@ pub(crate) fn mk_simd_types() -> TokenStream { let as_array_mut_op = generic_op_name("as_array_mut", ty); let from_bytes_op = generic_op_name("cvt_from_bytes", ty); let to_bytes_op = generic_op_name("cvt_to_bytes", ty); - let bytes = VecType::new(ScalarType::Unsigned, 8, align).rust(); + let bytes = ty.bytes_ty().rust(); let mask = ty.mask_ty().rust(); if ty.scalar == ScalarType::Mask { @@ -399,7 +399,6 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { } let mask_ty = ty.mask_ty().rust(); let block_ty = ty.block_ty().rust(); - let indices_ty = ty.indices_ty().rust(); let block_splat_body = match ty.n_bits() { 128 => quote! { block @@ -437,7 +436,6 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { const N: usize = #len; type Mask = #mask_ty; type Block = #block_ty; - type Indices = #indices_ty; type Array = [#scalar; #len]; #[inline(always)] @@ -491,7 +489,7 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { } #[inline(always)] - fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { self.simd.#swizzle_dyn_within_blocks_op(self, indices.simd_into(self.simd)) } } diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index 43d7388d1..fdfa52108 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -490,7 +490,7 @@ impl Level for WasmSimd128 { "wide swizzles should use the generic split implementation" ); - let bytes_ty = vec_ty.indices_ty(); + let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 4d65b8171..85244e4d0 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -2465,7 +2465,7 @@ impl X86 { } pub(crate) fn handle_swizzle_dyn_within_blocks(&self, op: Op, vec_ty: &VecType) -> TokenStream { - let bytes_ty = vec_ty.indices_ty(); + let bytes_ty = vec_ty.bytes_ty(); let bytes = bytes_ty.rust(); let wrapper = bytes_ty.aligned_wrapper(); let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 0972ae6f2..55c6c825d 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -308,8 +308,8 @@ impl Op { } OpSig::Slide { .. } => (vec![vec.clone(), vec.clone()], vec), OpSig::SwizzleDynWithinBlocks => { - let indices_ty = vec_ty.indices_ty().rust(); - (vec![vec.clone(), quote! { #indices_ty<#simd_ty> }], vec) + let bytes_ty = vec_ty.bytes_ty().rust(); + (vec![vec.clone(), quote! { #bytes_ty<#simd_ty> }], vec) } OpSig::Cvt { target_ty, @@ -431,7 +431,7 @@ impl Op { OpSig::SwizzleDynWithinBlocks => { let arg0 = &arg_names[0]; let arg1 = &arg_names[1]; - quote! { (#arg0, #arg1: impl SimdInto) -> Self } + quote! { (#arg0, #arg1: impl SimdInto) -> Self } } OpSig::Compare => { let arg0 = &arg_names[0]; diff --git a/fearless_simd_gen/src/types.rs b/fearless_simd_gen/src/types.rs index a1a7d1996..be1362143 100644 --- a/fearless_simd_gen/src/types.rs +++ b/fearless_simd_gen/src/types.rs @@ -157,7 +157,7 @@ impl VecType { Self::new(self.scalar, self.scalar_bits, 128 / self.scalar_bits) } - pub(crate) fn indices_ty(&self) -> Self { + pub(crate) fn bytes_ty(&self) -> Self { Self::new(ScalarType::Unsigned, 8, self.n_bits() / 8) } diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index d0f21b72d..d0050aac7 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -4150,7 +4150,7 @@ fn swizzle_dyn_within_blocks_f64x8(simd: S) { #[simd_test] fn swizzle_dyn_within_blocks_generic_indices(simd: S) { #[inline(always)] - fn do_swizzle>(value: V, indices: V::Indices) -> V { + fn do_swizzle>(value: V, indices: V::Bytes) -> V { value.swizzle_dyn_within_blocks(indices) }