|
| 1 | +/** |
| 2 | + * This code is released under the |
| 3 | + * Apache License Version 2.0 http://www.apache.org/licenses/. |
| 4 | + * |
| 5 | + * (c) Daniel Lemire |
| 6 | + */ |
| 7 | + |
| 8 | +/** |
| 9 | + * Native ARM NEON implementations of the (small) subset of x86 SSE intrinsics |
| 10 | + * used by FastPFOR. This replaces the SIMDe emulation layer on aarch64 targets: |
| 11 | + * every operation below maps directly to native NEON instructions. |
| 12 | + * |
| 13 | + * The mappings follow the well-known SSE->NEON correspondences (the same ones |
| 14 | + * used by projects such as sse2neon). Shift-by-amount operations are expressed |
| 15 | + * with the NEON variable-shift instruction so they accept both compile-time |
| 16 | + * constants (which the compiler folds to immediate shifts) and runtime counts. |
| 17 | + * |
| 18 | + * This header is valid in both C99 and C++ so it can be shared by the C codecs |
| 19 | + * (streamvbyte.c, varintdecode.c) and the C++ headers. |
| 20 | + */ |
| 21 | +#ifndef FASTPFOR_NEON_H_ |
| 22 | +#define FASTPFOR_NEON_H_ |
| 23 | + |
| 24 | +#if !(defined(__aarch64__) || (defined(_MSC_VER) && defined(_M_ARM64))) |
| 25 | +#error "fastpfor_neon.h is only for ARM (aarch64 / ARM64) targets" |
| 26 | +#endif |
| 27 | + |
| 28 | +#include <arm_neon.h> |
| 29 | +#include <stddef.h> |
| 30 | +#include <stdint.h> |
| 31 | + |
| 32 | +typedef int64x2_t __m128i; |
| 33 | +typedef float32x4_t __m128; |
| 34 | + |
| 35 | +/* ----------------------------- load / store ----------------------------- */ |
| 36 | + |
| 37 | +static inline __m128i _mm_loadu_si128(const __m128i *p) { |
| 38 | + return vld1q_s64((const int64_t *)p); |
| 39 | +} |
| 40 | +static inline __m128i _mm_lddqu_si128(const __m128i *p) { |
| 41 | + return vld1q_s64((const int64_t *)p); |
| 42 | +} |
| 43 | +static inline __m128i _mm_load_si128(const __m128i *p) { |
| 44 | + return vld1q_s64((const int64_t *)p); |
| 45 | +} |
| 46 | +static inline __m128i _mm_loadl_epi64(const __m128i *p) { |
| 47 | + return vcombine_s64(vld1_s64((const int64_t *)p), vdup_n_s64(0)); |
| 48 | +} |
| 49 | +static inline void _mm_storeu_si128(__m128i *p, __m128i a) { |
| 50 | + vst1q_s64((int64_t *)p, a); |
| 51 | +} |
| 52 | +static inline void _mm_store_si128(__m128i *p, __m128i a) { |
| 53 | + vst1q_s64((int64_t *)p, a); |
| 54 | +} |
| 55 | +static inline void _mm_stream_si128(__m128i *p, __m128i a) { |
| 56 | + vst1q_s64((int64_t *)p, a); |
| 57 | +} |
| 58 | +static inline void _mm_storel_epi64(__m128i *p, __m128i a) { |
| 59 | + vst1_s64((int64_t *)p, vget_low_s64(a)); |
| 60 | +} |
| 61 | + |
| 62 | +/* --------------------------------- set ---------------------------------- */ |
| 63 | + |
| 64 | +static inline __m128i _mm_setzero_si128(void) { return vdupq_n_s64(0); } |
| 65 | +static inline __m128i _mm_set1_epi32(int a) { |
| 66 | + return vreinterpretq_s64_s32(vdupq_n_s32(a)); |
| 67 | +} |
| 68 | +static inline __m128i _mm_set1_epi16(short a) { |
| 69 | + return vreinterpretq_s64_s16(vdupq_n_s16(a)); |
| 70 | +} |
| 71 | +static inline __m128i _mm_set1_epi8(signed char a) { |
| 72 | + return vreinterpretq_s64_s8(vdupq_n_s8(a)); |
| 73 | +} |
| 74 | +static inline __m128i _mm_set_epi64x(int64_t e1, int64_t e0) { |
| 75 | + int64_t d[2]; |
| 76 | + d[0] = e0; |
| 77 | + d[1] = e1; |
| 78 | + return vld1q_s64(d); |
| 79 | +} |
| 80 | +static inline __m128i |
| 81 | +_mm_set_epi8(signed char e15, signed char e14, signed char e13, signed char e12, |
| 82 | + signed char e11, signed char e10, signed char e9, signed char e8, |
| 83 | + signed char e7, signed char e6, signed char e5, signed char e4, |
| 84 | + signed char e3, signed char e2, signed char e1, signed char e0) { |
| 85 | + int8_t d[16]; |
| 86 | + d[0] = e0; d[1] = e1; d[2] = e2; d[3] = e3; |
| 87 | + d[4] = e4; d[5] = e5; d[6] = e6; d[7] = e7; |
| 88 | + d[8] = e8; d[9] = e9; d[10] = e10; d[11] = e11; |
| 89 | + d[12] = e12; d[13] = e13; d[14] = e14; d[15] = e15; |
| 90 | + return vreinterpretq_s64_s8(vld1q_s8(d)); |
| 91 | +} |
| 92 | +static inline __m128i |
| 93 | +_mm_setr_epi8(signed char e0, signed char e1, signed char e2, signed char e3, |
| 94 | + signed char e4, signed char e5, signed char e6, signed char e7, |
| 95 | + signed char e8, signed char e9, signed char e10, signed char e11, |
| 96 | + signed char e12, signed char e13, signed char e14, |
| 97 | + signed char e15) { |
| 98 | + int8_t d[16]; |
| 99 | + d[0] = e0; d[1] = e1; d[2] = e2; d[3] = e3; |
| 100 | + d[4] = e4; d[5] = e5; d[6] = e6; d[7] = e7; |
| 101 | + d[8] = e8; d[9] = e9; d[10] = e10; d[11] = e11; |
| 102 | + d[12] = e12; d[13] = e13; d[14] = e14; d[15] = e15; |
| 103 | + return vreinterpretq_s64_s8(vld1q_s8(d)); |
| 104 | +} |
| 105 | +static inline __m128i _mm_setr_epi16(short e0, short e1, short e2, short e3, |
| 106 | + short e4, short e5, short e6, short e7) { |
| 107 | + int16_t d[8]; |
| 108 | + d[0] = e0; d[1] = e1; d[2] = e2; d[3] = e3; |
| 109 | + d[4] = e4; d[5] = e5; d[6] = e6; d[7] = e7; |
| 110 | + return vreinterpretq_s64_s16(vld1q_s16(d)); |
| 111 | +} |
| 112 | + |
| 113 | +/* ----------------------------- bitwise / arith -------------------------- */ |
| 114 | + |
| 115 | +static inline __m128i _mm_and_si128(__m128i a, __m128i b) { |
| 116 | + return vandq_s64(a, b); |
| 117 | +} |
| 118 | +static inline __m128i _mm_or_si128(__m128i a, __m128i b) { |
| 119 | + return vorrq_s64(a, b); |
| 120 | +} |
| 121 | +static inline __m128i _mm_add_epi32(__m128i a, __m128i b) { |
| 122 | + return vreinterpretq_s64_s32( |
| 123 | + vaddq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); |
| 124 | +} |
| 125 | +static inline __m128i _mm_sub_epi32(__m128i a, __m128i b) { |
| 126 | + return vreinterpretq_s64_s32( |
| 127 | + vsubq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); |
| 128 | +} |
| 129 | +static inline __m128i _mm_mullo_epi32(__m128i a, __m128i b) { |
| 130 | + return vreinterpretq_s64_s32( |
| 131 | + vmulq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); |
| 132 | +} |
| 133 | +static inline __m128i _mm_mullo_epi16(__m128i a, __m128i b) { |
| 134 | + return vreinterpretq_s64_s16( |
| 135 | + vmulq_s16(vreinterpretq_s16_s64(a), vreinterpretq_s16_s64(b))); |
| 136 | +} |
| 137 | + |
| 138 | +/* --------------------------------- shifts ------------------------------- */ |
| 139 | +/* Variable-shift form: accepts runtime counts; the compiler lowers a |
| 140 | + * constant count to a native immediate shift. A right shift is a left shift |
| 141 | + * by a negative amount (NEON semantics); counts >= element width yield 0, |
| 142 | + * matching SSE. */ |
| 143 | + |
| 144 | +static inline __m128i _mm_slli_epi32(__m128i a, int imm) { |
| 145 | + return vreinterpretq_s64_u32( |
| 146 | + vshlq_u32(vreinterpretq_u32_s64(a), vdupq_n_s32(imm))); |
| 147 | +} |
| 148 | +static inline __m128i _mm_srli_epi32(__m128i a, int imm) { |
| 149 | + return vreinterpretq_s64_u32( |
| 150 | + vshlq_u32(vreinterpretq_u32_s64(a), vdupq_n_s32(-imm))); |
| 151 | +} |
| 152 | +static inline __m128i _mm_srli_epi16(__m128i a, int imm) { |
| 153 | + return vreinterpretq_s64_u16( |
| 154 | + vshlq_u16(vreinterpretq_u16_s64(a), vdupq_n_s16((int16_t)-imm))); |
| 155 | +} |
| 156 | +static inline __m128i _mm_slli_epi64(__m128i a, int imm) { |
| 157 | + return vreinterpretq_s64_u64( |
| 158 | + vshlq_u64(vreinterpretq_u64_s64(a), vdupq_n_s64(imm))); |
| 159 | +} |
| 160 | +static inline __m128i _mm_srli_epi64(__m128i a, int imm) { |
| 161 | + return vreinterpretq_s64_u64( |
| 162 | + vshlq_u64(vreinterpretq_u64_s64(a), vdupq_n_s64(-imm))); |
| 163 | +} |
| 164 | + |
| 165 | +/* Whole-register byte shifts. The byte count is always a compile-time |
| 166 | + * constant in FastPFOR, so vextq_u8 (which needs an immediate) is used. */ |
| 167 | +#define _mm_srli_si128(a, imm) \ |
| 168 | + vreinterpretq_s64_u8( \ |
| 169 | + vextq_u8(vreinterpretq_u8_s64(a), vdupq_n_u8(0), (imm))) |
| 170 | +#define _mm_slli_si128(a, imm) \ |
| 171 | + vreinterpretq_s64_u8( \ |
| 172 | + vextq_u8(vdupq_n_u8(0), vreinterpretq_u8_s64(a), (16 - (imm)))) |
| 173 | + |
| 174 | +/* ------------------------------- compares ------------------------------- */ |
| 175 | + |
| 176 | +static inline __m128i _mm_cmplt_epi32(__m128i a, __m128i b) { |
| 177 | + return vreinterpretq_s64_u32( |
| 178 | + vcltq_s32(vreinterpretq_s32_s64(a), vreinterpretq_s32_s64(b))); |
| 179 | +} |
| 180 | +static inline __m128i _mm_cmpeq_epi8(__m128i a, __m128i b) { |
| 181 | + return vreinterpretq_s64_u8( |
| 182 | + vceqq_u8(vreinterpretq_u8_s64(a), vreinterpretq_u8_s64(b))); |
| 183 | +} |
| 184 | + |
| 185 | +/* ------------------------------- shuffles ------------------------------- */ |
| 186 | + |
| 187 | +static inline __m128i _mm_shuffle_epi8(__m128i a, __m128i b) { |
| 188 | + /* pshufb: a byte of the index with its high bit set produces 0. Masking the |
| 189 | + * index with 0x8F leaves the low nibble and the high bit; vqtbl1q_u8 then |
| 190 | + * yields 0 for any index >= 16 (i.e. when the high bit was set). */ |
| 191 | + uint8x16_t tbl = vreinterpretq_u8_s64(a); |
| 192 | + uint8x16_t idx = vandq_u8(vreinterpretq_u8_s64(b), vdupq_n_u8(0x8F)); |
| 193 | + return vreinterpretq_s64_u8(vqtbl1q_u8(tbl, idx)); |
| 194 | +} |
| 195 | +static inline __m128i _mm_shuffle_epi32(__m128i a, const int imm) { |
| 196 | + /* Permute the four 32-bit lanes. `imm` is a compile-time constant at every |
| 197 | + * call site, so the broadcast fast paths fold away and the general case |
| 198 | + * builds a constant byte index that compiles to a single TBL. */ |
| 199 | + uint32x4_t v = vreinterpretq_u32_s64(a); |
| 200 | + if (imm == 0x00) |
| 201 | + return vreinterpretq_s64_u32(vdupq_laneq_u32(v, 0)); |
| 202 | + if (imm == 0x55) |
| 203 | + return vreinterpretq_s64_u32(vdupq_laneq_u32(v, 1)); |
| 204 | + if (imm == 0xAA) |
| 205 | + return vreinterpretq_s64_u32(vdupq_laneq_u32(v, 2)); |
| 206 | + if (imm == 0xFF) |
| 207 | + return vreinterpretq_s64_u32(vdupq_laneq_u32(v, 3)); |
| 208 | + { |
| 209 | + const uint8_t b0 = (uint8_t)((imm & 3) * 4); |
| 210 | + const uint8_t b1 = (uint8_t)(((imm >> 2) & 3) * 4); |
| 211 | + const uint8_t b2 = (uint8_t)(((imm >> 4) & 3) * 4); |
| 212 | + const uint8_t b3 = (uint8_t)(((imm >> 6) & 3) * 4); |
| 213 | + const uint8x16_t idx = { |
| 214 | + b0, (uint8_t)(b0 + 1), (uint8_t)(b0 + 2), (uint8_t)(b0 + 3), |
| 215 | + b1, (uint8_t)(b1 + 1), (uint8_t)(b1 + 2), (uint8_t)(b1 + 3), |
| 216 | + b2, (uint8_t)(b2 + 1), (uint8_t)(b2 + 2), (uint8_t)(b2 + 3), |
| 217 | + b3, (uint8_t)(b3 + 1), (uint8_t)(b3 + 2), (uint8_t)(b3 + 3)}; |
| 218 | + return vreinterpretq_s64_u8(vqtbl1q_u8(vreinterpretq_u8_s64(a), idx)); |
| 219 | + } |
| 220 | +} |
| 221 | +static inline __m128i _mm_blend_epi16(__m128i a, __m128i b, const int imm) { |
| 222 | + /* Per-16-bit-lane select from `a` (0) or `b` (1). `imm` is constant at every |
| 223 | + * call site, so this compound-literal mask folds to a constant vector load |
| 224 | + * feeding a single BSL, with no stack round-trip. */ |
| 225 | + const uint16x8_t mask = { |
| 226 | + (imm & 0x01) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 227 | + (imm & 0x02) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 228 | + (imm & 0x04) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 229 | + (imm & 0x08) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 230 | + (imm & 0x10) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 231 | + (imm & 0x20) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 232 | + (imm & 0x40) ? (uint16_t)0xFFFF : (uint16_t)0, |
| 233 | + (imm & 0x80) ? (uint16_t)0xFFFF : (uint16_t)0}; |
| 234 | + return vreinterpretq_s64_u16(vbslq_u16(mask, vreinterpretq_u16_s64(b), |
| 235 | + vreinterpretq_u16_s64(a))); |
| 236 | +} |
| 237 | + |
| 238 | +/* --------------------------- extract / convert -------------------------- */ |
| 239 | + |
| 240 | +#define _mm_extract_epi32(a, imm) \ |
| 241 | + vgetq_lane_s32(vreinterpretq_s32_s64(a), (imm)) |
| 242 | + |
| 243 | +static inline int _mm_cvtsi128_si32(__m128i a) { |
| 244 | + return vgetq_lane_s32(vreinterpretq_s32_s64(a), 0); |
| 245 | +} |
| 246 | +static inline __m128i _mm_cvtepu8_epi16(__m128i a) { |
| 247 | + return vreinterpretq_s64_u16(vmovl_u8(vget_low_u8(vreinterpretq_u8_s64(a)))); |
| 248 | +} |
| 249 | +static inline __m128i _mm_cvtepu16_epi32(__m128i a) { |
| 250 | + return vreinterpretq_s64_u32( |
| 251 | + vmovl_u16(vget_low_u16(vreinterpretq_u16_s64(a)))); |
| 252 | +} |
| 253 | +static inline __m128i _mm_cvtepi8_epi32(__m128i a) { |
| 254 | + int16x8_t t16 = vmovl_s8(vget_low_s8(vreinterpretq_s8_s64(a))); |
| 255 | + return vreinterpretq_s64_s32(vmovl_s16(vget_low_s16(t16))); |
| 256 | +} |
| 257 | + |
| 258 | +/* -------------------------------- masks --------------------------------- */ |
| 259 | + |
| 260 | +static inline int _mm_movemask_epi8(__m128i a) { |
| 261 | + uint8x16_t input = vreinterpretq_u8_s64(a); |
| 262 | + uint16x8_t high_bits = vreinterpretq_u16_u8(vshrq_n_u8(input, 7)); |
| 263 | + uint32x4_t paired16 = |
| 264 | + vreinterpretq_u32_u16(vsraq_n_u16(high_bits, high_bits, 7)); |
| 265 | + uint64x2_t paired32 = |
| 266 | + vreinterpretq_u64_u32(vsraq_n_u32(paired16, paired16, 14)); |
| 267 | + uint8x16_t paired64 = |
| 268 | + vreinterpretq_u8_u64(vsraq_n_u64(paired32, paired32, 28)); |
| 269 | + return vgetq_lane_u8(paired64, 0) | ((int)vgetq_lane_u8(paired64, 8) << 8); |
| 270 | +} |
| 271 | + |
| 272 | +/* --------------------------------- float -------------------------------- */ |
| 273 | + |
| 274 | +static inline __m128 _mm_castsi128_ps(__m128i a) { |
| 275 | + return vreinterpretq_f32_s64(a); |
| 276 | +} |
| 277 | +static inline int _mm_movemask_ps(__m128 a) { |
| 278 | + static const int32_t shifts[4] = {0, 1, 2, 3}; |
| 279 | + uint32x4_t signs = vshrq_n_u32(vreinterpretq_u32_f32(a), 31); |
| 280 | + uint32x4_t weighted = vshlq_u32(signs, vld1q_s32(shifts)); |
| 281 | + return (int)vaddvq_u32(weighted); |
| 282 | +} |
| 283 | + |
| 284 | +#endif /* FASTPFOR_NEON_H_ */ |
0 commit comments