|
| 1 | +#ifndef HALIDE_DECOMPOSE_VECTOR_SHUFFLE_H |
| 2 | +#define HALIDE_DECOMPOSE_VECTOR_SHUFFLE_H |
| 3 | + |
| 4 | +/** \file |
| 5 | + * |
| 6 | + * Perform vector shuffle by decomposing the operation to |
| 7 | + * a sequence of the sub shuffle steps where each step is a shuffle of: |
| 8 | + * - One or two slices as input (slice_a and slice_b) |
| 9 | + * - Produce one slice (dst slice) |
| 10 | + * - All the slices have the same length as target native vector (vl) |
| 11 | + * |
| 12 | + * The structure of the sequence of steps consists of: |
| 13 | + * 1. Outer loop to iterate the slices of dst vector. |
| 14 | + * 2. Inner loop to iterate the native shuffle steps to complete a single dst slice. |
| 15 | + * This can be multiple steps because a single native shuffle can take |
| 16 | + * only 2 slices (native vector length x 2) at most, while we may need |
| 17 | + * to fetch from wider location in the src vector. |
| 18 | + * |
| 19 | + * The following example, log of test code, illustrates how it works. |
| 20 | + * |
| 21 | + * src_lanes: 17, dst_lanes: 7, vl: 4 |
| 22 | + * input a: [0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100, 110, 120, 130, 140, 150, 160, ] |
| 23 | + * input b: [170, 180, 190, 200, 210, 220, 230, 240, 250, 260, 270, 280, 290, 300, 310, 320, 330, ] |
| 24 | + * indices: [6, 13, 24, 14, 7, 11, 5, ] |
| 25 | + * |
| 26 | + * slice a:[40, 50, 60, 70, ], slice b:[120, 130, 140, 150, ], indices:[2, 5, -1, 6, ] |
| 27 | + * => slice output:[60, 130, -559038801, 140, ] |
| 28 | + * slice a:[60, 130, -559038801, 140, ], slice b:[210, 220, 230, 240, ], indices:[0, 1, 7, 3, ] |
| 29 | + * => slice output:[60, 130, 240, 140, ] |
| 30 | + * slice a:[40, 50, 60, 70, ], slice b:[80, 90, 100, 110, ], indices:[3, 7, 1, -1, ] |
| 31 | + * => slice output:[70, 110, 50, -559038801, ] |
| 32 | + * |
| 33 | + * output: [60, 130, 240, 140, 70, 110, 50, ] |
| 34 | + * |
| 35 | + */ |
| 36 | + |
| 37 | +#include "Error.h" |
| 38 | +#include "Util.h" |
| 39 | + |
| 40 | +#include <optional> |
| 41 | +#include <vector> |
| 42 | + |
| 43 | +namespace Halide { |
| 44 | +namespace Internal { |
| 45 | + |
| 46 | +/** Enum to represent the special cases of slice index */ |
| 47 | +enum { |
| 48 | + SliceIndexNone = -1, |
| 49 | + SliceIndexCarryPrevResult = -2, |
| 50 | +}; |
| 51 | + |
| 52 | +struct NativeShuffle { |
| 53 | + int slice_a; |
| 54 | + int slice_b; |
| 55 | + std::vector<int> lane_map; |
| 56 | + |
| 57 | + NativeShuffle(int vl, int a, int b) |
| 58 | + : slice_a(a), slice_b(b) { |
| 59 | + lane_map.resize(vl, SliceIndexNone); |
| 60 | + } |
| 61 | +}; |
| 62 | + |
| 63 | +std::vector<std::vector<NativeShuffle>> decompose_to_native_shuffles( |
| 64 | + int src_lanes, const std::vector<int> &indices, int vl); |
| 65 | + |
| 66 | +/** Algorithm logic for shuffle decomposition, parameterized on vector type |
| 67 | + * and a codegen-like class that provides primitive vector operations. |
| 68 | + */ |
| 69 | +template<typename CodeGenTy, typename VecTy> |
| 70 | +struct DecomposeVectorShuffle { |
| 71 | + // TODO: when upgrading to C++20, replace with a concept. |
| 72 | + // get_vector_num_elements may be overloaded (e.g. on Type* and Value*), so use |
| 73 | + // expression SFINAE rather than a method pointer to handle overload resolution. |
| 74 | + static_assert(std::is_convertible_v<decltype(std::declval<CodeGenTy &>().get_vector_num_elements(std::declval<VecTy>())), int>, |
| 75 | + "CodeGenTy must provide: int get_vector_num_elements(VecTy)"); |
| 76 | + static_assert(std::is_invocable_r_v<VecTy, decltype(&CodeGenTy::slice_vector), CodeGenTy &, const VecTy &, int, int>, |
| 77 | + "CodeGenTy must provide: VecTy slice_vector(const VecTy &, int, int)"); |
| 78 | + static_assert(std::is_invocable_r_v<VecTy, decltype(&CodeGenTy::concat_vectors), CodeGenTy &, const std::vector<VecTy> &>, |
| 79 | + "CodeGenTy must provide: VecTy concat_vectors(const std::vector<VecTy> &)"); |
| 80 | + static_assert(std::is_invocable_r_v<VecTy, decltype(&CodeGenTy::shuffle_scalable_vectors_general), CodeGenTy &, |
| 81 | + const VecTy &, const VecTy &, const std::vector<int> &>, |
| 82 | + "CodeGenTy must provide: VecTy shuffle_scalable_vectors_general(const VecTy &, const VecTy &, const std::vector<int> &)"); |
| 83 | + static_assert(std::is_invocable_r_v<VecTy, decltype(&CodeGenTy::create_undef_vector_like), CodeGenTy &, const VecTy &, int>, |
| 84 | + "CodeGenTy must provide: VecTy create_undef_vector_like(const VecTy &, int)"); |
| 85 | + |
| 86 | + DecomposeVectorShuffle(CodeGenTy &codegen, const VecTy &src_a, const VecTy &src_b, int src_lanes, int vl) |
| 87 | + : codegen(codegen), |
| 88 | + vl(vl), |
| 89 | + src_a(align_up_vector(src_a, vl)), |
| 90 | + src_b(align_up_vector(src_b, vl)), |
| 91 | + src_lanes(src_lanes), |
| 92 | + src_lanes_aligned(align_up(src_lanes, vl)) { |
| 93 | + } |
| 94 | + |
| 95 | + VecTy run(const std::vector<int> &indices) { |
| 96 | + auto shuffle_plan = decompose_to_native_shuffles(src_lanes, indices, vl); |
| 97 | + int dst_lanes = static_cast<int>(indices.size()); |
| 98 | + |
| 99 | + // process each block divided by vl |
| 100 | + std::vector<VecTy> shuffled_dst_slices; |
| 101 | + shuffled_dst_slices.reserve(shuffle_plan.size()); |
| 102 | + |
| 103 | + for (const auto &steps_for_dst_slice : shuffle_plan) { |
| 104 | + std::optional<VecTy> dst_slice = std::nullopt; |
| 105 | + for (const auto &step : steps_for_dst_slice) { |
| 106 | + // Obtain 1st slice a |
| 107 | + VecTy a; |
| 108 | + if (step.slice_a == SliceIndexCarryPrevResult) { |
| 109 | + internal_assert(dst_slice.has_value()) << "Tried to carry from undefined previous result"; |
| 110 | + a = *dst_slice; |
| 111 | + } else { |
| 112 | + a = get_vl_slice(step.slice_a); |
| 113 | + } |
| 114 | + // Obtain 2nd slice b |
| 115 | + std::optional<VecTy> b; |
| 116 | + if (step.slice_b == SliceIndexNone) { |
| 117 | + b = std::nullopt; |
| 118 | + } else { |
| 119 | + b = std::optional<VecTy>(get_vl_slice(step.slice_b)); |
| 120 | + } |
| 121 | + // Perform shuffle where vector length is aligned |
| 122 | + dst_slice = codegen.shuffle_scalable_vectors_general(a, b.value_or(VecTy{}), step.lane_map); |
| 123 | + } |
| 124 | + if (!dst_slice.has_value()) { |
| 125 | + // No shuffle step for this slice, i.e. all the indices are -1 |
| 126 | + dst_slice = codegen.create_undef_vector_like(src_a, vl); |
| 127 | + } |
| 128 | + shuffled_dst_slices.push_back(*dst_slice); |
| 129 | + } |
| 130 | + |
| 131 | + return codegen.slice_vector(codegen.concat_vectors(shuffled_dst_slices), 0, dst_lanes); |
| 132 | + } |
| 133 | + |
| 134 | +private: |
| 135 | + // Helper to extract slice with lanes=vl |
| 136 | + VecTy get_vl_slice(int slice_index) { |
| 137 | + const int num_slices_a = src_lanes_aligned / vl; |
| 138 | + int start_index = slice_index * vl; |
| 139 | + if (slice_index < num_slices_a) { |
| 140 | + return codegen.slice_vector(src_a, start_index, vl); |
| 141 | + } else { |
| 142 | + start_index -= src_lanes_aligned; |
| 143 | + return codegen.slice_vector(src_b, start_index, vl); |
| 144 | + } |
| 145 | + } |
| 146 | + |
| 147 | + VecTy align_up_vector(const VecTy &v, int align) { |
| 148 | + int len = codegen.get_vector_num_elements(v); |
| 149 | + return codegen.slice_vector(v, 0, align_up(len, align)); |
| 150 | + } |
| 151 | + |
| 152 | + CodeGenTy &codegen; |
| 153 | + int vl; |
| 154 | + VecTy src_a; |
| 155 | + VecTy src_b; |
| 156 | + int src_lanes; |
| 157 | + int src_lanes_aligned; |
| 158 | +}; |
| 159 | + |
| 160 | +} // namespace Internal |
| 161 | +} // namespace Halide |
| 162 | + |
| 163 | +#endif |
0 commit comments