diff --git a/fearless_simd/src/generated/avx2.rs b/fearless_simd/src/generated/avx2.rs index 3f4ee93e..c306f88e 100644 --- a/fearless_simd/src/generated/avx2.rs +++ b/fearless_simd/src/generated/avx2.rs @@ -175,6 +175,20 @@ impl Simd for Avx2 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x4, indices: u8x16) -> f32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -697,6 +711,20 @@ impl Simd for Avx2 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x16, indices: u8x16) -> i8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1065,6 +1093,20 @@ impl Simd for Avx2 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x16, indices: u8x16) -> u8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1606,6 +1648,20 @@ impl Simd for Avx2 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x8, indices: u8x16) -> i16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -1955,6 +2011,20 @@ impl Simd for Avx2 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x8, indices: u8x16) -> u16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2475,6 +2545,20 @@ impl Simd for Avx2 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x4, indices: u8x16) -> i32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -2844,6 +2928,20 @@ impl Simd for Avx2 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x4, indices: u8x16) -> u32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3380,6 +3478,20 @@ impl Simd for Avx2 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x2, indices: u8x16) -> f64x2 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -3973,6 +4085,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f32x8, indices: u8x32) -> f32x8 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_f32x8(a).val.0, indices.into()); + token.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -4558,6 +4684,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i8x32, indices: u8x32) -> i8x32 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_i8x32(a).val.0, indices.into()); + token.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -5014,6 +5154,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u8x32, indices: u8x32) -> u8x32 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_u8x32(a).val.0, indices.into()); + token.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -5657,6 +5811,25 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i16x16, indices: u8x32) -> i16x16 { + let result = + _mm256_shuffle_epi8(token.cvt_to_bytes_i16x16(a).val.0, indices.into()); + token.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -6096,6 +6269,25 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u16x16, indices: u8x32) -> u16x16 { + let result = + _mm256_shuffle_epi8(token.cvt_to_bytes_u16x16(a).val.0, indices.into()); + token.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -6737,6 +6929,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: i32x8, indices: u8x32) -> i32x8 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_i32x8(a).val.0, indices.into()); + token.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -7172,6 +7378,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: u32x8, indices: u8x32) -> u32x8 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_u32x8(a).val.0, indices.into()); + token.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -7786,6 +8006,20 @@ impl Simd for Avx2 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx2, a: f64x4, indices: u8x32) -> f64x4 { + let result = _mm256_shuffle_epi8(token.cvt_to_bytes_f64x4(a).val.0, indices.into()); + token.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -8420,6 +8654,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -8859,6 +9106,15 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -9130,6 +9386,15 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -9628,6 +9893,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -9908,6 +10186,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -10414,6 +10705,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -10690,6 +10994,19 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -11166,6 +11483,15 @@ impl Simd for Avx2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/avx512.rs b/fearless_simd/src/generated/avx512.rs index 976ebd2a..d2e015fe 100644 --- a/fearless_simd/src/generated/avx512.rs +++ b/fearless_simd/src/generated/avx512.rs @@ -183,6 +183,21 @@ impl Simd for Avx512 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = token.cvt_to_bytes_f32x4(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -702,6 +717,21 @@ impl Simd for Avx512 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = token.cvt_to_bytes_i8x16(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1148,6 +1178,21 @@ impl Simd for Avx512 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = token.cvt_to_bytes_u8x16(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1719,6 +1764,21 @@ impl Simd for Avx512 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = token.cvt_to_bytes_i16x8(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -2116,6 +2176,21 @@ impl Simd for Avx512 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = token.cvt_to_bytes_u16x8(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2638,6 +2713,21 @@ impl Simd for Avx512 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = token.cvt_to_bytes_i32x4(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -3035,6 +3125,21 @@ impl Simd for Avx512 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = token.cvt_to_bytes_u32x4(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3548,6 +3653,21 @@ impl Simd for Avx512 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = token.cvt_to_bytes_f64x2(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -4148,6 +4268,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x8, indices: u8x32) -> f32x8 { + let bytes = token.cvt_to_bytes_f32x8(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_f32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { crate::kernel!( #[inline(always)] @@ -4768,6 +4903,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x32, indices: u8x32) -> i8x32 { + let bytes = token.cvt_to_bytes_i8x32(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_i8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { crate::kernel!( #[inline(always)] @@ -5325,6 +5475,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x32, indices: u8x32) -> u8x32 { + let bytes = token.cvt_to_bytes_u8x32(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_u8x32(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { crate::kernel!( #[inline(always)] @@ -6017,6 +6182,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x16, indices: u8x32) -> i16x16 { + let bytes = token.cvt_to_bytes_i16x16(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_i16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { crate::kernel!( #[inline(always)] @@ -6505,6 +6689,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x16, indices: u8x32) -> u16x16 { + let bytes = token.cvt_to_bytes_u16x16(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_u16x16(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { crate::kernel!( #[inline(always)] @@ -7142,6 +7345,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x8, indices: u8x32) -> i32x8 { + let bytes = token.cvt_to_bytes_i32x8(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_i32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { crate::kernel!( #[inline(always)] @@ -7618,6 +7836,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x8, indices: u8x32) -> u32x8 { + let bytes = token.cvt_to_bytes_u32x8(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_u32x8(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { crate::kernel!( #[inline(always)] @@ -8224,6 +8457,21 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x4, indices: u8x32) -> f64x4 { + let bytes = token.cvt_to_bytes_f64x4(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + token.cvt_from_bytes_f64x4(u8x32 { + val: crate::support::Aligned256(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { crate::kernel!( #[inline(always)] @@ -8890,6 +9138,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f32x16, indices: u8x64) -> f32x16 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_f32x16(a).val.0, indices.into()); + token.cvt_from_bytes_f32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { crate::kernel!( #[inline(always)] @@ -9554,6 +9821,20 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i8x64, indices: u8x64) -> i8x64 { + let result = _mm512_shuffle_epi8(token.cvt_to_bytes_i8x64(a).val.0, indices.into()); + token.cvt_from_bytes_i8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { crate::kernel!( #[inline(always)] @@ -10127,6 +10408,20 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u8x64, indices: u8x64) -> u8x64 { + let result = _mm512_shuffle_epi8(token.cvt_to_bytes_u8x64(a).val.0, indices.into()); + token.cvt_from_bytes_u8x64(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { crate::kernel!( #[inline(always)] @@ -10849,6 +11144,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i16x32, indices: u8x64) -> i16x32 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_i16x32(a).val.0, indices.into()); + token.cvt_from_bytes_i16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { crate::kernel!( #[inline(always)] @@ -11347,6 +11661,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u16x32, indices: u8x64) -> u16x32 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_u16x32(a).val.0, indices.into()); + token.cvt_from_bytes_u16x32(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { crate::kernel!( #[inline(always)] @@ -12022,6 +12355,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: i32x16, indices: u8x64) -> i32x16 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_i32x16(a).val.0, indices.into()); + token.cvt_from_bytes_i32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { crate::kernel!( #[inline(always)] @@ -12512,6 +12864,25 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: u32x16, indices: u8x64) -> u32x16 { + let result = + _mm512_shuffle_epi8(token.cvt_to_bytes_u32x16(a).val.0, indices.into()); + token.cvt_from_bytes_u32x16(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { crate::kernel!( #[inline(always)] @@ -13153,6 +13524,20 @@ impl Simd for Avx512 { }) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Avx512, a: f64x8, indices: u8x64) -> f64x8 { + let result = _mm512_shuffle_epi8(token.cvt_to_bytes_f64x8(a).val.0, indices.into()); + token.cvt_from_bytes_f64x8(u8x64 { + val: crate::support::Aligned512(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { crate::kernel!( #[inline(always)] diff --git a/fearless_simd/src/generated/fallback.rs b/fearless_simd/src/generated/fallback.rs index 1024b172..34f28fcd 100644 --- a/fearless_simd/src/generated/fallback.rs +++ b/fearless_simd/src/generated/fallback.rs @@ -215,6 +215,78 @@ impl Simd for Fallback { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let bytes = self.cvt_to_bytes_f32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_f32x4(result) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { [ f32::abs(a[0usize]), @@ -611,6 +683,78 @@ impl Simd for Fallback { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let bytes = self.cvt_to_bytes_i8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_i8x16(result) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { [ i8::wrapping_add(a[0usize], b[0usize]), @@ -1236,6 +1380,78 @@ impl Simd for Fallback { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let bytes = self.cvt_to_bytes_u8x16(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_u8x16(result) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { [ u8::wrapping_add(a[0usize], b[0usize]), @@ -2187,6 +2403,78 @@ impl Simd for Fallback { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let bytes = self.cvt_to_bytes_i16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_i16x8(result) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { [ i16::wrapping_add(a[0usize], b[0usize]), @@ -2613,6 +2901,78 @@ impl Simd for Fallback { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let bytes = self.cvt_to_bytes_u16x8(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_u16x8(result) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { [ u16::wrapping_add(a[0usize], b[0usize]), @@ -3243,6 +3603,78 @@ impl Simd for Fallback { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let bytes = self.cvt_to_bytes_i32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_i32x4(result) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { [ i32::wrapping_add(a[0usize], b[0usize]), @@ -3571,6 +4003,78 @@ impl Simd for Fallback { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let bytes = self.cvt_to_bytes_u32x4(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_u32x4(result) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { [ u32::wrapping_add(a[0usize], b[0usize]), @@ -4035,6 +4539,78 @@ impl Simd for Fallback { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let bytes = self.cvt_to_bytes_f64x2(a); + let result: u8x16 = [ + { + let index = indices[0usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[1usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[2usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[3usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[4usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[5usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[6usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[7usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[8usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[9usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[10usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[11usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[12usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[13usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[14usize] as usize; + bytes[index % 16usize] + }, + { + let index = indices[15usize] as usize; + bytes[index % 16usize] + }, + ] + .simd_into(self); + self.cvt_from_bytes_f64x2(result) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { [f64::abs(a[0usize]), f64::abs(a[1usize])].simd_into(self) } @@ -4441,6 +5017,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4798,6 +5383,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -5062,6 +5656,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -5440,6 +6043,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5704,6 +6320,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -6104,6 +6733,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -6373,6 +7011,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6748,6 +7395,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -7177,6 +7833,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -7557,6 +8226,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -7814,6 +8492,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -8258,6 +8945,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -8524,6 +9224,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -8953,6 +9666,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -9215,6 +9941,19 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -9606,6 +10345,15 @@ impl Simd for Fallback { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/neon.rs b/fearless_simd/src/generated/neon.rs index 8553ff66..3af22020 100644 --- a/fearless_simd/src/generated/neon.rs +++ b/fearless_simd/src/generated/neon.rs @@ -165,6 +165,20 @@ impl Simd for Neon { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f32x4, indices: u8x16) -> f32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -642,6 +656,20 @@ impl Simd for Neon { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i8x16, indices: u8x16) -> i8x16 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1008,6 +1036,20 @@ impl Simd for Neon { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u8x16, indices: u8x16) -> u8x16 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1553,6 +1595,20 @@ impl Simd for Neon { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i16x8, indices: u8x16) -> i16x8 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -1919,6 +1975,20 @@ impl Simd for Neon { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u16x8, indices: u8x16) -> u16x8 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2454,6 +2524,20 @@ impl Simd for Neon { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: i32x4, indices: u8x16) -> i32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -2830,6 +2914,20 @@ impl Simd for Neon { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: u32x4, indices: u8x16) -> u32x4 { + let result = vqtbl1q_u8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3364,6 +3462,20 @@ impl Simd for Neon { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Neon, a: f64x2, indices: u8x16) -> f64x2 { + let result = vqtbl1q_u8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -3981,6 +4093,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4375,6 +4496,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4676,6 +4806,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -5098,6 +5237,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5399,6 +5551,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -5834,6 +5999,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -6140,6 +6314,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6559,6 +6742,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -7050,6 +7242,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -7461,6 +7666,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -7771,6 +7985,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -8196,6 +8419,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -8515,6 +8751,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -8962,6 +9211,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -9277,6 +9539,19 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -9704,6 +9979,15 @@ impl Simd for Neon { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/simd_trait.rs b/fearless_simd/src/generated/simd_trait.rs index 7690bca3..324b8457 100644 --- a/fearless_simd/src/generated/simd_trait.rs +++ b/fearless_simd/src/generated/simd_trait.rs @@ -144,6 +144,8 @@ pub trait Simd: a: f32x4, b: f32x4, ) -> f32x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4; #[doc = "Compute the absolute value of each element."] fn abs_f32x4(self, a: f32x4) -> f32x4; #[doc = "Negate each element of the vector."] @@ -252,6 +254,8 @@ pub trait Simd: a: i8x16, b: i8x16, ) -> i8x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -336,6 +340,8 @@ pub trait Simd: a: u8x16, b: u8x16, ) -> u8x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -457,6 +463,8 @@ pub trait Simd: a: i16x8, b: i16x8, ) -> i16x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -541,6 +549,8 @@ pub trait Simd: a: u16x8, b: u16x8, ) -> u16x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -662,6 +672,8 @@ pub trait Simd: a: i32x4, b: i32x4, ) -> i32x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -748,6 +760,8 @@ pub trait Simd: a: u32x4, b: u32x4, ) -> u32x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -869,6 +883,8 @@ pub trait Simd: a: f64x2, b: f64x2, ) -> f64x2; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2; #[doc = "Compute the absolute value of each element."] fn abs_f64x2(self, a: f64x2) -> f64x2; #[doc = "Negate each element of the vector."] @@ -1002,6 +1018,8 @@ pub trait Simd: a: f32x8, b: f32x8, ) -> f32x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8; #[doc = "Compute the absolute value of each element."] fn abs_f32x8(self, a: f32x8) -> f32x8; #[doc = "Negate each element of the vector."] @@ -1112,6 +1130,8 @@ pub trait Simd: a: i8x32, b: i8x32, ) -> i8x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1198,6 +1218,8 @@ pub trait Simd: a: u8x32, b: u8x32, ) -> u8x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1323,6 +1345,12 @@ pub trait Simd: a: i16x16, b: i16x16, ) -> i16x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1409,6 +1437,12 @@ pub trait Simd: a: u16x16, b: u16x16, ) -> u16x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1536,6 +1570,8 @@ pub trait Simd: a: i32x8, b: i32x8, ) -> i32x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1624,6 +1660,8 @@ pub trait Simd: a: u32x8, b: u32x8, ) -> u32x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -1749,6 +1787,8 @@ pub trait Simd: a: f64x4, b: f64x4, ) -> f64x4; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4; #[doc = "Compute the absolute value of each element."] fn abs_f64x4(self, a: f64x4) -> f64x4; #[doc = "Negate each element of the vector."] @@ -1886,6 +1926,12 @@ pub trait Simd: a: f32x16, b: f32x16, ) -> f32x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16; #[doc = "Compute the absolute value of each element."] fn abs_f32x16(self, a: f32x16) -> f32x16; #[doc = "Negate each element of the vector."] @@ -1998,6 +2044,8 @@ pub trait Simd: a: i8x64, b: i8x64, ) -> i8x64; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2082,6 +2130,8 @@ pub trait Simd: a: u8x64, b: u8x64, ) -> u8x64; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2205,6 +2255,12 @@ pub trait Simd: a: i16x32, b: i16x32, ) -> i16x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2289,6 +2345,12 @@ pub trait Simd: a: u16x32, b: u16x32, ) -> u16x32; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2416,6 +2478,12 @@ pub trait Simd: a: i32x16, b: i32x16, ) -> i32x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2502,6 +2570,12 @@ pub trait Simd: a: u32x16, b: u32x16, ) -> u32x16; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16; #[doc = "Add two vectors element-wise, wrapping on overflow."] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16; #[doc = "Subtract two vectors element-wise, wrapping on overflow."] @@ -2627,6 +2701,8 @@ pub trait Simd: a: f64x8, b: f64x8, ) -> f64x8; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8; #[doc = "Compute the absolute value of each element."] fn abs_f64x8(self, a: f64x8) -> f64x8; #[doc = "Negate each element of the vector."] @@ -2837,6 +2913,8 @@ pub trait SimdBase: fn slide(self, rhs: impl SimdInto) -> Self; #[doc = "Like `slide`, but operates independently on each 128-bit block."] fn slide_within_blocks(self, rhs: impl SimdInto) -> Self; + #[doc = "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\nThe `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\nOut-of-range index behavior varies by platform."] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self; } #[doc = r" Functionality implemented by floating-point SIMD vectors."] pub trait SimdFloat: diff --git a/fearless_simd/src/generated/simd_types.rs b/fearless_simd/src/generated/simd_types.rs index c05fa1b7..3964c796 100644 --- a/fearless_simd/src/generated/simd_types.rs +++ b/fearless_simd/src/generated/simd_types.rs @@ -130,6 +130,11 @@ impl SimdBase for f32x4 { self.simd .slide_within_blocks_f32x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x4 { #[inline(always)] @@ -383,6 +388,11 @@ impl SimdBase for i8x16 { self.simd .slide_within_blocks_i8x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x16 { #[inline(always)] @@ -568,6 +578,11 @@ impl SimdBase for u8x16 { self.simd .slide_within_blocks_u8x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u8x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x16 { #[inline(always)] @@ -844,6 +859,11 @@ impl SimdBase for i16x8 { self.simd .slide_within_blocks_i16x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x8 { #[inline(always)] @@ -1029,6 +1049,11 @@ impl SimdBase for u16x8 { self.simd .slide_within_blocks_u16x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u16x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x8 { #[inline(always)] @@ -1305,6 +1330,11 @@ impl SimdBase for i32x4 { self.simd .slide_within_blocks_i32x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x4 { #[inline(always)] @@ -1502,6 +1532,11 @@ impl SimdBase for u32x4 { self.simd .slide_within_blocks_u32x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u32x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x4 { #[inline(always)] @@ -1790,6 +1825,11 @@ impl SimdBase for f64x2 { self.simd .slide_within_blocks_f64x2::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f64x2(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x2 { #[inline(always)] @@ -2120,6 +2160,11 @@ impl SimdBase for f32x8 { self.simd .slide_within_blocks_f32x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x8 { #[inline(always)] @@ -2380,6 +2425,11 @@ impl SimdBase for i8x32 { self.simd .slide_within_blocks_i8x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x32 { #[inline(always)] @@ -2572,6 +2622,11 @@ impl SimdBase for u8x32 { self.simd .slide_within_blocks_u8x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u8x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x32 { #[inline(always)] @@ -2860,6 +2915,11 @@ impl SimdBase for i16x16 { self.simd .slide_within_blocks_i16x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x16 { #[inline(always)] @@ -3058,6 +3118,11 @@ impl SimdBase for u16x16 { self.simd .slide_within_blocks_u16x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u16x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x16 { #[inline(always)] @@ -3342,6 +3407,11 @@ impl SimdBase for i32x8 { self.simd .slide_within_blocks_i32x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x8 { #[inline(always)] @@ -3546,6 +3616,11 @@ impl SimdBase for u32x8 { self.simd .slide_within_blocks_u32x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u32x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x8 { #[inline(always)] @@ -3841,6 +3916,11 @@ impl SimdBase for f64x4 { self.simd .slide_within_blocks_f64x4::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f64x4(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x4 { #[inline(always)] @@ -4184,6 +4264,11 @@ impl SimdBase for f32x16 { self.simd .slide_within_blocks_f32x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f32x16 { #[inline(always)] @@ -4439,6 +4524,11 @@ impl SimdBase for i8x64 { self.simd .slide_within_blocks_i8x64::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i8x64 { #[inline(always)] @@ -4625,6 +4715,11 @@ impl SimdBase for u8x64 { self.simd .slide_within_blocks_u8x64::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u8x64(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u8x64 { #[inline(always)] @@ -4907,6 +5002,11 @@ impl SimdBase for i16x32 { self.simd .slide_within_blocks_i16x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i16x32 { #[inline(always)] @@ -5099,6 +5199,11 @@ impl SimdBase for u16x32 { self.simd .slide_within_blocks_u16x32::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u16x32(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u16x32 { #[inline(always)] @@ -5382,6 +5487,11 @@ impl SimdBase for i32x16 { self.simd .slide_within_blocks_i32x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_i32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for i32x16 { #[inline(always)] @@ -5586,6 +5696,11 @@ impl SimdBase for u32x16 { self.simd .slide_within_blocks_u32x16::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_u32x16(self, indices.simd_into(self.simd)) + } } impl crate::SimdInt for u32x16 { #[inline(always)] @@ -5876,6 +5991,11 @@ impl SimdBase for f64x8 { self.simd .slide_within_blocks_f64x8::(self, rhs.simd_into(self.simd)) } + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd + .swizzle_dyn_within_blocks_f64x8(self, indices.simd_into(self.simd)) + } } impl crate::SimdFloat for f64x8 { #[inline(always)] diff --git a/fearless_simd/src/generated/sse4_2.rs b/fearless_simd/src/generated/sse4_2.rs index abbac0c5..5d0929e3 100644 --- a/fearless_simd/src/generated/sse4_2.rs +++ b/fearless_simd/src/generated/sse4_2.rs @@ -201,6 +201,20 @@ impl Simd for Sse4_2 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f32x4, indices: u8x16) -> f32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f32x4(a).val.0, indices.into()); + token.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { crate::kernel!( #[inline(always)] @@ -708,6 +722,20 @@ impl Simd for Sse4_2 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i8x16, indices: u8x16) -> i8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i8x16(a).val.0, indices.into()); + token.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { crate::kernel!( #[inline(always)] @@ -1073,6 +1101,20 @@ impl Simd for Sse4_2 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u8x16, indices: u8x16) -> u8x16 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u8x16(a).val.0, indices.into()); + token.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { crate::kernel!( #[inline(always)] @@ -1611,6 +1653,20 @@ impl Simd for Sse4_2 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i16x8, indices: u8x16) -> i16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i16x8(a).val.0, indices.into()); + token.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { crate::kernel!( #[inline(always)] @@ -1957,6 +2013,20 @@ impl Simd for Sse4_2 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u16x8, indices: u8x16) -> u16x8 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u16x8(a).val.0, indices.into()); + token.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { crate::kernel!( #[inline(always)] @@ -2471,6 +2541,20 @@ impl Simd for Sse4_2 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: i32x4, indices: u8x16) -> i32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_i32x4(a).val.0, indices.into()); + token.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { crate::kernel!( #[inline(always)] @@ -2825,6 +2909,20 @@ impl Simd for Sse4_2 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: u32x4, indices: u8x16) -> u32x4 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_u32x4(a).val.0, indices.into()); + token.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { crate::kernel!( #[inline(always)] @@ -3343,6 +3441,20 @@ impl Simd for Sse4_2 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + crate::kernel!( + #[inline(always)] + fn kernel(token: Sse4_2, a: f64x2, indices: u8x16) -> f64x2 { + let result = _mm_shuffle_epi8(token.cvt_to_bytes_f64x2(a).val.0, indices.into()); + token.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: token, + }) + } + ); + kernel(self, a, indices) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { crate::kernel!( #[inline(always)] @@ -3906,6 +4018,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -4277,6 +4398,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -4555,6 +4685,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -4952,6 +5091,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -5230,6 +5382,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -5648,6 +5813,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -5931,6 +6105,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -6325,6 +6508,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -6773,6 +6965,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -7212,6 +7417,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -7483,6 +7697,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -7987,6 +8210,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -8267,6 +8503,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -8761,6 +9010,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -9037,6 +9299,19 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -9492,6 +9767,15 @@ impl Simd for Sse4_2 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd/src/generated/wasm.rs b/fearless_simd/src/generated/wasm.rs index 09a2c004..8161c57c 100644 --- a/fearless_simd/src/generated/wasm.rs +++ b/fearless_simd/src/generated/wasm.rs @@ -157,6 +157,14 @@ impl Simd for WasmSimd128 { self.slide_f32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x4(self, a: f32x4, indices: u8x16) -> f32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_f32x4(a).val.0, indices.into()); + self.cvt_from_bytes_f32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f32x4(self, a: f32x4) -> f32x4 { f32x4_abs(a.into()).simd_into(self) } @@ -451,6 +459,14 @@ impl Simd for WasmSimd128 { self.slide_i8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x16(self, a: i8x16, indices: u8x16) -> i8x16 { + let result = u8x16_swizzle(self.cvt_to_bytes_i8x16(a).val.0, indices.into()); + self.cvt_from_bytes_i8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i8x16(self, a: i8x16, b: i8x16) -> i8x16 { i8x16_add(a.into(), b.into()).simd_into(self) } @@ -664,6 +680,14 @@ impl Simd for WasmSimd128 { self.slide_u8x16::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x16(self, a: u8x16, indices: u8x16) -> u8x16 { + let result = u8x16_swizzle(self.cvt_to_bytes_u8x16(a).val.0, indices.into()); + self.cvt_from_bytes_u8x16(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u8x16(self, a: u8x16, b: u8x16) -> u8x16 { u8x16_add(a.into(), b.into()).simd_into(self) } @@ -974,6 +998,14 @@ impl Simd for WasmSimd128 { self.slide_i16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x8(self, a: i16x8, indices: u8x16) -> i16x8 { + let result = u8x16_swizzle(self.cvt_to_bytes_i16x8(a).val.0, indices.into()); + self.cvt_from_bytes_i16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i16x8(self, a: i16x8, b: i16x8) -> i16x8 { i16x8_add(a.into(), b.into()).simd_into(self) } @@ -1171,6 +1203,14 @@ impl Simd for WasmSimd128 { self.slide_u16x8::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x8(self, a: u16x8, indices: u8x16) -> u16x8 { + let result = u8x16_swizzle(self.cvt_to_bytes_u16x8(a).val.0, indices.into()); + self.cvt_from_bytes_u16x8(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u16x8(self, a: u16x8, b: u16x8) -> u16x8 { u16x8_add(a.into(), b.into()).simd_into(self) } @@ -1461,6 +1501,14 @@ impl Simd for WasmSimd128 { self.slide_i32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x4(self, a: i32x4, indices: u8x16) -> i32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_i32x4(a).val.0, indices.into()); + self.cvt_from_bytes_i32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_i32x4(self, a: i32x4, b: i32x4) -> i32x4 { i32x4_add(a.into(), b.into()).simd_into(self) } @@ -1662,6 +1710,14 @@ impl Simd for WasmSimd128 { self.slide_u32x4::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x4(self, a: u32x4, indices: u8x16) -> u32x4 { + let result = u8x16_swizzle(self.cvt_to_bytes_u32x4(a).val.0, indices.into()); + self.cvt_from_bytes_u32x4(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn add_u32x4(self, a: u32x4, b: u32x4) -> u32x4 { u32x4_add(a.into(), b.into()).simd_into(self) } @@ -1952,6 +2008,14 @@ impl Simd for WasmSimd128 { self.slide_f64x2::(a, b) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x2(self, a: f64x2, indices: u8x16) -> f64x2 { + let result = u8x16_swizzle(self.cvt_to_bytes_f64x2(a).val.0, indices.into()); + self.cvt_from_bytes_f64x2(u8x16 { + val: crate::support::Aligned128(result), + simd: self, + }) + } + #[inline(always)] fn abs_f64x2(self, a: f64x2) -> f64x2 { f64x2_abs(a.into()).simd_into(self) } @@ -2307,6 +2371,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x8(self, a: f32x8, indices: u8x32) -> f32x8 { + let (a0, a1) = self.split_f32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f32x4( + self.swizzle_dyn_within_blocks_f32x4(a0, indices0), + self.swizzle_dyn_within_blocks_f32x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x8(self, a: f32x8) -> f32x8 { let (a0, a1) = self.split_f32x8(a); self.combine_f32x4(self.abs_f32x4(a0), self.abs_f32x4(a1)) @@ -2677,6 +2750,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x32(self, a: i8x32, indices: u8x32) -> i8x32 { + let (a0, a1) = self.split_i8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i8x16( + self.swizzle_dyn_within_blocks_i8x16(a0, indices0), + self.swizzle_dyn_within_blocks_i8x16(a1, indices1), + ) + } + #[inline(always)] fn add_i8x32(self, a: i8x32, b: i8x32) -> i8x32 { let (a0, a1) = self.split_i8x32(a); let (b0, b1) = self.split_i8x32(b); @@ -2954,6 +3036,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x32(self, a: u8x32, indices: u8x32) -> u8x32 { + let (a0, a1) = self.split_u8x32(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u8x16( + self.swizzle_dyn_within_blocks_u8x16(a0, indices0), + self.swizzle_dyn_within_blocks_u8x16(a1, indices1), + ) + } + #[inline(always)] fn add_u8x32(self, a: u8x32, b: u8x32) -> u8x32 { let (a0, a1) = self.split_u8x32(a); let (b0, b1) = self.split_u8x32(b); @@ -3350,6 +3441,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x16( + self, + a: i16x16, + indices: u8x32, + ) -> i16x16 { + let (a0, a1) = self.split_i16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i16x8( + self.swizzle_dyn_within_blocks_i16x8(a0, indices0), + self.swizzle_dyn_within_blocks_i16x8(a1, indices1), + ) + } + #[inline(always)] fn add_i16x16(self, a: i16x16, b: i16x16) -> i16x16 { let (a0, a1) = self.split_i16x16(a); let (b0, b1) = self.split_i16x16(b); @@ -3627,6 +3731,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x16( + self, + a: u16x16, + indices: u8x32, + ) -> u16x16 { + let (a0, a1) = self.split_u16x16(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u16x8( + self.swizzle_dyn_within_blocks_u16x8(a0, indices0), + self.swizzle_dyn_within_blocks_u16x8(a1, indices1), + ) + } + #[inline(always)] fn add_u16x16(self, a: u16x16, b: u16x16) -> u16x16 { let (a0, a1) = self.split_u16x16(a); let (b0, b1) = self.split_u16x16(b); @@ -4032,6 +4149,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x8(self, a: i32x8, indices: u8x32) -> i32x8 { + let (a0, a1) = self.split_i32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_i32x4( + self.swizzle_dyn_within_blocks_i32x4(a0, indices0), + self.swizzle_dyn_within_blocks_i32x4(a1, indices1), + ) + } + #[inline(always)] fn add_i32x8(self, a: i32x8, b: i32x8) -> i32x8 { let (a0, a1) = self.split_i32x8(a); let (b0, b1) = self.split_i32x8(b); @@ -4314,6 +4440,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x8(self, a: u32x8, indices: u8x32) -> u32x8 { + let (a0, a1) = self.split_u32x8(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_u32x4( + self.swizzle_dyn_within_blocks_u32x4(a0, indices0), + self.swizzle_dyn_within_blocks_u32x4(a1, indices1), + ) + } + #[inline(always)] fn add_u32x8(self, a: u32x8, b: u32x8) -> u32x8 { let (a0, a1) = self.split_u32x8(a); let (b0, b1) = self.split_u32x8(b); @@ -4707,6 +4842,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x4(self, a: f64x4, indices: u8x32) -> f64x4 { + let (a0, a1) = self.split_f64x4(a); + let (indices0, indices1) = self.split_u8x32(indices); + self.combine_f64x2( + self.swizzle_dyn_within_blocks_f64x2(a0, indices0), + self.swizzle_dyn_within_blocks_f64x2(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x4(self, a: f64x4) -> f64x4 { let (a0, a1) = self.split_f64x4(a); self.combine_f64x2(self.abs_f64x2(a0), self.abs_f64x2(a1)) @@ -5154,6 +5298,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f32x16( + self, + a: f32x16, + indices: u8x64, + ) -> f32x16 { + let (a0, a1) = self.split_f32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f32x8( + self.swizzle_dyn_within_blocks_f32x8(a0, indices0), + self.swizzle_dyn_within_blocks_f32x8(a1, indices1), + ) + } + #[inline(always)] fn abs_f32x16(self, a: f32x16) -> f32x16 { let (a0, a1) = self.split_f32x16(a); self.combine_f32x8(self.abs_f32x8(a0), self.abs_f32x8(a1)) @@ -5563,6 +5720,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i8x64(self, a: i8x64, indices: u8x64) -> i8x64 { + let (a0, a1) = self.split_i8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i8x32( + self.swizzle_dyn_within_blocks_i8x32(a0, indices0), + self.swizzle_dyn_within_blocks_i8x32(a1, indices1), + ) + } + #[inline(always)] fn add_i8x64(self, a: i8x64, b: i8x64) -> i8x64 { let (a0, a1) = self.split_i8x64(a); let (b0, b1) = self.split_i8x64(b); @@ -5833,6 +5999,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u8x64(self, a: u8x64, indices: u8x64) -> u8x64 { + let (a0, a1) = self.split_u8x64(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u8x32( + self.swizzle_dyn_within_blocks_u8x32(a0, indices0), + self.swizzle_dyn_within_blocks_u8x32(a1, indices1), + ) + } + #[inline(always)] fn add_u8x64(self, a: u8x64, b: u8x64) -> u8x64 { let (a0, a1) = self.split_u8x64(a); let (b0, b1) = self.split_u8x64(b); @@ -6280,6 +6455,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i16x32( + self, + a: i16x32, + indices: u8x64, + ) -> i16x32 { + let (a0, a1) = self.split_i16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i16x16( + self.swizzle_dyn_within_blocks_i16x16(a0, indices0), + self.swizzle_dyn_within_blocks_i16x16(a1, indices1), + ) + } + #[inline(always)] fn add_i16x32(self, a: i16x32, b: i16x32) -> i16x32 { let (a0, a1) = self.split_i16x32(a); let (b0, b1) = self.split_i16x32(b); @@ -6559,6 +6747,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u16x32( + self, + a: u16x32, + indices: u8x64, + ) -> u16x32 { + let (a0, a1) = self.split_u16x32(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u16x16( + self.swizzle_dyn_within_blocks_u16x16(a0, indices0), + self.swizzle_dyn_within_blocks_u16x16(a1, indices1), + ) + } + #[inline(always)] fn add_u16x32(self, a: u16x32, b: u16x32) -> u16x32 { let (a0, a1) = self.split_u16x32(a); let (b0, b1) = self.split_u16x32(b); @@ -7004,6 +7205,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_i32x16( + self, + a: i32x16, + indices: u8x64, + ) -> i32x16 { + let (a0, a1) = self.split_i32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_i32x8( + self.swizzle_dyn_within_blocks_i32x8(a0, indices0), + self.swizzle_dyn_within_blocks_i32x8(a1, indices1), + ) + } + #[inline(always)] fn add_i32x16(self, a: i32x16, b: i32x16) -> i32x16 { let (a0, a1) = self.split_i32x16(a); let (b0, b1) = self.split_i32x16(b); @@ -7279,6 +7493,19 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_u32x16( + self, + a: u32x16, + indices: u8x64, + ) -> u32x16 { + let (a0, a1) = self.split_u32x16(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_u32x8( + self.swizzle_dyn_within_blocks_u32x8(a0, indices0), + self.swizzle_dyn_within_blocks_u32x8(a1, indices1), + ) + } + #[inline(always)] fn add_u32x16(self, a: u32x16, b: u32x16) -> u32x16 { let (a0, a1) = self.split_u32x16(a); let (b0, b1) = self.split_u32x16(b); @@ -7704,6 +7931,15 @@ impl Simd for WasmSimd128 { ) } #[inline(always)] + fn swizzle_dyn_within_blocks_f64x8(self, a: f64x8, indices: u8x64) -> f64x8 { + let (a0, a1) = self.split_f64x8(a); + let (indices0, indices1) = self.split_u8x64(indices); + self.combine_f64x4( + self.swizzle_dyn_within_blocks_f64x4(a0, indices0), + self.swizzle_dyn_within_blocks_f64x4(a1, indices1), + ) + } + #[inline(always)] fn abs_f64x8(self, a: f64x8) -> f64x8 { let (a0, a1) = self.split_f64x8(a); self.combine_f64x4(self.abs_f64x4(a0), self.abs_f64x4(a1)) diff --git a/fearless_simd_gen/src/generic.rs b/fearless_simd_gen/src/generic.rs index 96b710c0..485fcbbd 100644 --- a/fearless_simd_gen/src/generic.rs +++ b/fearless_simd_gen/src/generic.rs @@ -69,6 +69,20 @@ pub(crate) fn generic_op(op: &Op, ty: &VecType) -> TokenStream { } } } + OpSig::SwizzleDynWithinBlocks => { + let bytes_ty = ty.bytes_ty(); + let split_indices = generic_op_name("split", &bytes_ty); + quote! { + #method_sig { + let (a0, a1) = self.#split(a); + let (indices0, indices1) = self.#split_indices(indices); + self.#combine( + self.#do_half(a0, indices0), + self.#do_half(a1, indices1), + ) + } + } + } OpSig::Ternary => { quote! { #method_sig { diff --git a/fearless_simd_gen/src/mk_fallback.rs b/fearless_simd_gen/src/mk_fallback.rs index 92099258..bf511662 100644 --- a/fearless_simd_gen/src/mk_fallback.rs +++ b/fearless_simd_gen/src/mk_fallback.rs @@ -392,6 +392,38 @@ impl Level for Fallback { } } } + OpSig::SwizzleDynWithinBlocks => { + assert_eq!( + vec_ty.n_bits(), + self.native_width(), + "wide swizzles should use the generic split implementation" + ); + let bytes_ty = vec_ty.bytes_ty(); + let bytes_rust = bytes_ty.rust(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + let byte_count = bytes_ty.len; + let items = make_list( + (0..byte_count) + .map(|idx| { + quote! { + { + let index = indices[#idx] as usize; + bytes[index % #byte_count] + } + } + }) + .collect::>(), + ); + + quote! { + #method_sig { + let bytes = self.#to_bytes(a); + let result: #bytes_rust = #items.simd_into(self); + self.#from_bytes(result) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_neon.rs b/fearless_simd_gen/src/mk_neon.rs index 2cdf737f..0eb203ff 100644 --- a/fearless_simd_gen/src/mk_neon.rs +++ b/fearless_simd_gen/src/mk_neon.rs @@ -424,6 +424,26 @@ impl Level for Neon { } } } + OpSig::SwizzleDynWithinBlocks => { + assert_eq!( + vec_ty.n_bits(), + self.native_width(), + "wide swizzles should use the generic split implementation" + ); + + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + self.kernel_method(op, vec_ty, |token| { + quote! { + let result = vqtbl1q_u8(#token.#to_bytes(a).val.0, indices.into()); + #token.#from_bytes(#bytes { val: #wrapper(result), simd: #token }) + } + }) + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_simd_types.rs b/fearless_simd_gen/src/mk_simd_types.rs index b6f2aafc..8ca4d2bf 100644 --- a/fearless_simd_gen/src/mk_simd_types.rs +++ b/fearless_simd_gen/src/mk_simd_types.rs @@ -32,7 +32,7 @@ pub(crate) fn mk_simd_types() -> TokenStream { let as_array_mut_op = generic_op_name("as_array_mut", ty); let from_bytes_op = generic_op_name("cvt_from_bytes", ty); let to_bytes_op = generic_op_name("cvt_to_bytes", ty); - let bytes = VecType::new(ScalarType::Unsigned, 8, align).rust(); + let bytes = ty.bytes_ty().rust(); let mask = ty.mask_ty().rust(); if ty.scalar == ScalarType::Mask { @@ -429,6 +429,7 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { let as_array_mut_op = generic_op_name("as_array_mut", ty); let slide_op = generic_op_name("slide", ty); let slide_blockwise_op = generic_op_name("slide_within_blocks", ty); + let swizzle_dyn_within_blocks_op = generic_op_name("swizzle_dyn_within_blocks", ty); quote! { impl SimdBase for #name { type Element = #scalar; @@ -486,6 +487,11 @@ fn simd_vec_impl(ty: &VecType) -> TokenStream { fn slide_within_blocks(self, rhs: impl SimdInto) -> Self { self.simd.#slide_blockwise_op::(self, rhs.simd_into(self.simd)) } + + #[inline(always)] + fn swizzle_dyn_within_blocks(self, indices: impl SimdInto) -> Self { + self.simd.#swizzle_dyn_within_blocks_op(self, indices.simd_into(self.simd)) + } } impl crate::#vec_trait_id for #name { #( #methods )* diff --git a/fearless_simd_gen/src/mk_wasm.rs b/fearless_simd_gen/src/mk_wasm.rs index f62014bd..fdfa5210 100644 --- a/fearless_simd_gen/src/mk_wasm.rs +++ b/fearless_simd_gen/src/mk_wasm.rs @@ -483,6 +483,26 @@ impl Level for WasmSimd128 { } } } + OpSig::SwizzleDynWithinBlocks => { + assert_eq!( + vec_ty.n_bits(), + self.native_width(), + "wide swizzles should use the generic split implementation" + ); + + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + quote! { + #method_sig { + let result = u8x16_swizzle(self.#to_bytes(a).val.0, indices.into()); + self.#from_bytes(#bytes { val: #wrapper(result), simd: self }) + } + } + } OpSig::Cvt { target_ty, scalar_bits, diff --git a/fearless_simd_gen/src/mk_x86.rs b/fearless_simd_gen/src/mk_x86.rs index 724c12ef..85244e4d 100644 --- a/fearless_simd_gen/src/mk_x86.rs +++ b/fearless_simd_gen/src/mk_x86.rs @@ -306,6 +306,7 @@ impl Level for X86 { OpSig::Zip { select_low } => self.handle_zip(op, vec_ty, select_low), OpSig::Unzip { select_even } => self.handle_unzip(op, vec_ty, select_even), OpSig::Slide { granularity } => self.handle_slide(method_sig, vec_ty, granularity), + OpSig::SwizzleDynWithinBlocks => self.handle_swizzle_dyn_within_blocks(op, vec_ty), OpSig::Cvt { target_ty, scalar_bits, @@ -2463,6 +2464,43 @@ impl X86 { } } + pub(crate) fn handle_swizzle_dyn_within_blocks(&self, op: Op, vec_ty: &VecType) -> TokenStream { + let bytes_ty = vec_ty.bytes_ty(); + let bytes = bytes_ty.rust(); + let wrapper = bytes_ty.aligned_wrapper(); + let to_bytes = generic_op_name("cvt_to_bytes", vec_ty); + let from_bytes = generic_op_name("cvt_from_bytes", vec_ty); + + self.kernel_method(op, vec_ty, |token| { + let body = if *self == Self::Avx512 { + match vec_ty.n_bits() { + 128 => quote! { + let bytes = #token.#to_bytes(a).val.0; + let result = _mm_mask_shuffle_epi8(bytes, u16::MAX, bytes, indices.into()); + }, + 256 => quote! { + let bytes = #token.#to_bytes(a).val.0; + let result = _mm256_mask_shuffle_epi8(bytes, u32::MAX, bytes, indices.into()); + }, + 512 => quote! { + let result = _mm512_shuffle_epi8(#token.#to_bytes(a).val.0, indices.into()); + }, + _ => unreachable!(), + } + } else { + let shuffle = simple_sign_unaware_intrinsic("shuffle", &bytes_ty); + quote! { + let result = #shuffle(#token.#to_bytes(a).val.0, indices.into()); + } + }; + + quote! { + #body + #token.#from_bytes(#bytes { val: #wrapper(result), simd: #token }) + } + }) + } + pub(crate) fn handle_cvt( &self, op: Op, diff --git a/fearless_simd_gen/src/ops.rs b/fearless_simd_gen/src/ops.rs index 17495b41..55c6c825 100644 --- a/fearless_simd_gen/src/ops.rs +++ b/fearless_simd_gen/src/ops.rs @@ -82,6 +82,9 @@ pub(crate) enum OpSig { Deinterleave, /// Takes two arguments of a vector type, plus a const generic shift amount, and returns that same vector type. Slide { granularity: SlideGranularity }, + /// Takes a vector and a same-width byte-index vector, and returns the original vector type with its bytes + /// dynamically swizzled within each 128-bit block. + SwizzleDynWithinBlocks, /// Takes a single argument of the source vector type, and returns a vector type of the target scalar type and the /// same length. Cvt { @@ -304,6 +307,10 @@ impl Op { (vec![vec.clone(), vec.clone()], quote! { (#vec, #vec) }) } OpSig::Slide { .. } => (vec![vec.clone(), vec.clone()], vec), + OpSig::SwizzleDynWithinBlocks => { + let bytes_ty = vec_ty.bytes_ty().rust(); + (vec![vec.clone(), quote! { #bytes_ty<#simd_ty> }], vec) + } OpSig::Cvt { target_ty, scalar_bits, @@ -421,6 +428,11 @@ impl Op { let arg1 = &arg_names[1]; quote! { (#arg0, #arg1: impl SimdInto) -> Self } } + OpSig::SwizzleDynWithinBlocks => { + let arg0 = &arg_names[0]; + let arg1 = &arg_names[1]; + quote! { (#arg0, #arg1: impl SimdInto) -> Self } + } OpSig::Compare => { let arg0 = &arg_names[0]; let arg1 = &arg_names[1]; @@ -587,6 +599,14 @@ const BASE_OPS: &[Op] = &[ }, "Like `slide`, but operates independently on each 128-bit block.", ), + Op::new( + "swizzle_dyn_within_blocks", + OpKind::BaseTraitMethod, + OpSig::SwizzleDynWithinBlocks, + "Dynamically swizzle this vector's bytes independently within each 128-bit block.\n\n\ + The `indices` operand is a same-width byte vector. For each output byte, index values `0..=15` select the corresponding byte from the same 128-bit input block.\n\n\ + Out-of-range index behavior varies by platform.", + ), ]; const MASK_REPRESENTATION_OPS: &[Op] = &[ @@ -1611,6 +1631,7 @@ impl OpSig { | Self::AsArray { .. } | Self::FromBytes | Self::ToBytes => &["a"], + Self::SwizzleDynWithinBlocks => &["a", "indices"], Self::Binary | Self::Compare | Self::Combine { .. } @@ -1643,6 +1664,7 @@ impl OpSig { | Self::MaskReduce { .. } | Self::AsArray { .. } | Self::ToBytes => &["self"], + Self::SwizzleDynWithinBlocks => &["self", "indices"], Self::Binary | Self::Compare | Self::Zip { .. } @@ -1704,6 +1726,7 @@ impl OpSig { | Self::StoreArray | Self::FromBytes | Self::ToBytes + | Self::SwizzleDynWithinBlocks | Self::Slide { .. } => return None, }; Some(args) diff --git a/fearless_simd_gen/src/types.rs b/fearless_simd_gen/src/types.rs index 670d6a5c..be136214 100644 --- a/fearless_simd_gen/src/types.rs +++ b/fearless_simd_gen/src/types.rs @@ -157,6 +157,10 @@ impl VecType { Self::new(self.scalar, self.scalar_bits, 128 / self.scalar_bits) } + pub(crate) fn bytes_ty(&self) -> Self { + Self::new(ScalarType::Unsigned, 8, self.n_bits() / 8) + } + pub(crate) fn split_operand(&self) -> Option { if self.n_bits() <= 128 { return None; diff --git a/fearless_simd_tests/tests/harness/mod.rs b/fearless_simd_tests/tests/harness/mod.rs index c250667c..d0050aac 100644 --- a/fearless_simd_tests/tests/harness/mod.rs +++ b/fearless_simd_tests/tests/harness/mod.rs @@ -3794,6 +3794,388 @@ fn bitcast_native(simd: S) { ); } +fn expected_swizzle_within_blocks(bytes: [u8; N], indices: [u8; N]) -> [u8; N] { + assert_eq!(N % 16, 0); + core::array::from_fn(|i| { + let block_start = (i / 16) * 16; + bytes[block_start + usize::from(indices[i])] + }) +} + +#[expect(clippy::cast_possible_truncation, reason = "truncation is deliberate")] +fn swizzle_test_byte(i: usize) -> u8 { + (i * 37 + 11) as u8 +} + +fn swizzle_test_index(i: usize) -> u8 { + u8::try_from(15 - (i % 16)).unwrap() +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f32x4(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: f32x4 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i8x16(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: i8x16 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u8x16(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: u8x16 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i16x8(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: i16x8 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u16x8(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: u16x8 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i32x4(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: i32x4 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u32x4(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: u32x4 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f64x2(simd: S) { + let bytes: [u8; 16] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 16] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x16::simd_from(simd, bytes); + let index_vec = u8x16::simd_from(simd, indices); + let value: f64x2 = byte_vec.bitcast(); + let result_bytes: u8x16 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: f32x8 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i8x32(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: i8x32 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u8x32(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: u8x32 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i16x16(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: i16x16 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u16x16(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: u16x16 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: i32x8 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u32x8(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: u32x8 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f64x4(simd: S) { + let bytes: [u8; 32] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 32] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x32::simd_from(simd, bytes); + let index_vec = u8x32::simd_from(simd, indices); + let value: f64x4 = byte_vec.bitcast(); + let result_bytes: u8x32 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f32x16(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: f32x16 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i8x64(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: i8x64 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u8x64(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: u8x64 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i16x32(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: i16x32 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u16x32(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: u16x32 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_i32x16(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: i32x16 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_u32x16(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: u32x16 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_f64x8(simd: S) { + let bytes: [u8; 64] = core::array::from_fn(swizzle_test_byte); + let indices: [u8; 64] = core::array::from_fn(swizzle_test_index); + let expected = expected_swizzle_within_blocks(bytes, indices); + + let byte_vec = u8x64::simd_from(simd, bytes); + let index_vec = u8x64::simd_from(simd, indices); + let value: f64x8 = byte_vec.bitcast(); + let result_bytes: u8x64 = value.swizzle_dyn_within_blocks(index_vec).bitcast(); + + assert_eq!(*result_bytes, expected); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_generic_indices(simd: S) { + #[inline(always)] + fn do_swizzle>(value: V, indices: V::Bytes) -> V { + value.swizzle_dyn_within_blocks(indices) + } + + let value = u8x16::from_fn(simd, |i| u8::try_from(i).unwrap()); + let indices = u8x16::from_fn(simd, |i| u8::try_from(15 - i).unwrap()); + let result = do_swizzle::>(value, indices); + + assert_eq!( + *result, + [15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0] + ); +} + +#[simd_test] +fn swizzle_dyn_within_blocks_oob_does_not_panic(simd: S) { + let value = u8x16::from_fn(simd, |i| u8::try_from(i).unwrap()); + let indices = u8x16::simd_from( + simd, + [ + 16, 17, 31, 0x80, 0x8f, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, + ], + ); + let _ = value.swizzle_dyn_within_blocks(indices); +} + #[simd_test] fn wrapping_add_u32(simd: S) { assert_eq!(