|
| 1 | +// --------------------------------------------------------------------------------------- |
| 2 | +// ILGPU Algorithms |
| 3 | +// Copyright (c) 2019-2023 ILGPU Project |
| 4 | +// www.ilgpu.net |
| 5 | +// |
| 6 | +// File: RadixSortExtensions.Float4E2M1.cs |
| 7 | +// |
| 8 | +// This file is part of ILGPU and is distributed under the University of Illinois Open |
| 9 | +// Source License. See LICENSE.txt for details. |
| 10 | +// --------------------------------------------------------------------------------------- |
| 11 | + |
| 12 | +using ILGPU.Algorithms.RadixSortOperations; |
| 13 | +using ILGPU.Algorithms.ScanReduceOperations; |
| 14 | +using ILGPU.Runtime; |
| 15 | +using System.Runtime.CompilerServices; |
| 16 | + |
| 17 | +namespace ILGPU.Algorithms |
| 18 | +{ |
| 19 | + // WebGL Float4E2M1-key radix sort. FP4 is a 1-byte sub-word key (value in the low |
| 20 | + // nibble); the WebGL render-to-texture scatter writes WHOLE 32-bit texels and cannot |
| 21 | + // move a sub-texel value, so - exactly like Half, BFloat16 and FP8 (RadixSortExtensions.cs |
| 22 | + // / RadixSortExtensions.BFloat16.cs / RadixSortExtensions.Float8E4M3.cs) - FP4 sorts via |
| 23 | + // an UNPACKED f32 working representation: copy-in widens each Float4E2M1 to f32 (lossless: |
| 24 | + // every one of the 16 finite FP4 codes is a strict subset of f32), the radix bit is derived |
| 25 | + // by narrowing back to Float4E2M1 and calling the canonical ExtractRadixBits, and copy-out |
| 26 | + // narrows the sorted f32 back to Float4E2M1 (exact round-trip for any value that began as a |
| 27 | + // Float4E2M1). Mirrors the FP8 path one-for-one; the only difference is NumBits == 4 (the |
| 28 | + // FP4 value occupies the low nibble) rather than 8, so the per-bit loop runs 4 passes. |
| 29 | + static partial class RadixSortExtensions |
| 30 | + { |
| 31 | + private static void WebGLScatterRadixCopyInFloat4E2M1<TStride>( |
| 32 | + Index1D index, ArrayView1D<Float4E2M1, TStride> input, |
| 33 | + ArrayView1D<float, Stride1D.Dense> output) |
| 34 | + where TStride : struct, IStride1D => |
| 35 | + output[index.X] = (float)input[index.X]; |
| 36 | + |
| 37 | + private static void WebGLScatterRadixCopyOutFloat4E2M1<TStride>( |
| 38 | + Index1D index, ArrayView1D<float, Stride1D.Dense> input, |
| 39 | + ArrayView1D<Float4E2M1, TStride> output) |
| 40 | + where TStride : struct, IStride1D => |
| 41 | + output[index.X] = (Float4E2M1)input[index.X]; |
| 42 | + |
| 43 | + private static void WebGLScatterRadixExtractBitFloat4E2M1<TRadixSortOperation>( |
| 44 | + Index1D index, ArrayView1D<float, Stride1D.Dense> keys, |
| 45 | + ArrayView1D<int, Stride1D.Dense> flags, int bit) |
| 46 | + where TRadixSortOperation : struct, IRadixSortOperation<Float4E2M1> |
| 47 | + { |
| 48 | + TRadixSortOperation op = default; |
| 49 | + flags[index.X] = op.ExtractRadixBits((Float4E2M1)keys[index.X], bit, 1); |
| 50 | + } |
| 51 | + |
| 52 | + // Keys-only Float4E2M1 sort. Invoked by reflection from CreateRadixSort (the outer |
| 53 | + // method is generic on T; the compiler can't see T == Float4E2M1 to bind the |
| 54 | + // IRadixSortOperation<Float4E2M1> constraint statically). Called once per handler. |
| 55 | + private static RadixSort<Float4E2M1, TStride> CreateWebGLScatterRadixSortFloat4E2M1< |
| 56 | + TStride, TRadixSortOperation>(Accelerator accelerator, IScatterProvider scatter) |
| 57 | + where TStride : struct, IStride1D |
| 58 | + where TRadixSortOperation : struct, IRadixSortOperation<Float4E2M1> |
| 59 | + { |
| 60 | + var copyIn = accelerator.LoadAutoGroupedKernel< |
| 61 | + Index1D, ArrayView1D<Float4E2M1, TStride>, ArrayView1D<float, Stride1D.Dense>>( |
| 62 | + WebGLScatterRadixCopyInFloat4E2M1<TStride>); |
| 63 | + var copyOut = accelerator.LoadAutoGroupedKernel< |
| 64 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<Float4E2M1, TStride>>( |
| 65 | + WebGLScatterRadixCopyOutFloat4E2M1<TStride>); |
| 66 | + var extractBit = accelerator.LoadAutoGroupedKernel< |
| 67 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, int>( |
| 68 | + WebGLScatterRadixExtractBitFloat4E2M1<TRadixSortOperation>); |
| 69 | + var computeDest = accelerator.LoadAutoGroupedKernel< |
| 70 | + Index1D, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, |
| 71 | + ArrayView1D<int, Stride1D.Dense>, int>(WebGLScatterRadixComputeDest); |
| 72 | + var exclusiveScan = accelerator.CreateScan< |
| 73 | + int, Stride1D.Dense, Stride1D.Dense, AddInt32>(ScanKind.Exclusive); |
| 74 | + |
| 75 | + int numBits = default(TRadixSortOperation).NumBits; // 4 |
| 76 | + |
| 77 | + return (stream, view, temp) => |
| 78 | + { |
| 79 | + int n = (int)view.Length; |
| 80 | + if (n <= 1) |
| 81 | + return; |
| 82 | + |
| 83 | + using var keysA = accelerator.Allocate1D<float>(n); |
| 84 | + using var keysB = accelerator.Allocate1D<float>(n); |
| 85 | + using var flags = accelerator.Allocate1D<int>(n); |
| 86 | + using var onePrefix = accelerator.Allocate1D<int>(n); |
| 87 | + using var dest = accelerator.Allocate1D<int>(n); |
| 88 | + using var scanTemp = accelerator.Allocate1D<int>(1); |
| 89 | + |
| 90 | + copyIn(stream, n, view, keysA.View); |
| 91 | + |
| 92 | + var src = keysA; |
| 93 | + var dst = keysB; |
| 94 | + for (int bit = 0; bit < numBits; bit++) |
| 95 | + { |
| 96 | + extractBit(stream, n, src.View, flags.View, bit); |
| 97 | + exclusiveScan(stream, flags.View, onePrefix.View, scanTemp.View); |
| 98 | + computeDest(stream, n, flags.View, onePrefix.View, dest.View, n); |
| 99 | + scatter.Scatter(dst.View, src.View, dest.View, n, "float"); |
| 100 | + var tmp = src; src = dst; dst = tmp; |
| 101 | + } |
| 102 | + |
| 103 | + copyOut(stream, n, src.View, view); |
| 104 | + }; |
| 105 | + } |
| 106 | + |
| 107 | + |
| 108 | + // Float4E2M1-KEY pairs sort (FP4 key + any 4/8-byte non-FP4 value). Keys use the |
| 109 | + // unpacked f32 working representation; values use the same int/float/uint scatter |
| 110 | + // program as the generic pairs path. Invoked by reflection from CreateRadixSortPairs. |
| 111 | + private static RadixSortPairs<Float4E2M1, TKeyStride, TValue, TValueStride> |
| 112 | + CreateWebGLScatterRadixSortPairsFloat4E2M1Key< |
| 113 | + TKeyStride, TValue, TValueStride, TRadixSortOperation>( |
| 114 | + Accelerator accelerator, IScatterProvider scatter) |
| 115 | + where TKeyStride : struct, IStride1D |
| 116 | + where TValue : unmanaged |
| 117 | + where TValueStride : struct, IStride1D |
| 118 | + where TRadixSortOperation : struct, IRadixSortOperation<Float4E2M1> |
| 119 | + { |
| 120 | + var copyInKeys = accelerator.LoadAutoGroupedKernel< |
| 121 | + Index1D, ArrayView1D<Float4E2M1, TKeyStride>, ArrayView1D<float, Stride1D.Dense>>( |
| 122 | + WebGLScatterRadixCopyInFloat4E2M1<TKeyStride>); |
| 123 | + var copyOutKeys = accelerator.LoadAutoGroupedKernel< |
| 124 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<Float4E2M1, TKeyStride>>( |
| 125 | + WebGLScatterRadixCopyOutFloat4E2M1<TKeyStride>); |
| 126 | + var copyInVals = accelerator.LoadAutoGroupedKernel< |
| 127 | + Index1D, ArrayView1D<TValue, TValueStride>, ArrayView1D<TValue, Stride1D.Dense>>( |
| 128 | + WebGLScatterRadixCopyIn<TValue, TValueStride>); |
| 129 | + var copyOutVals = accelerator.LoadAutoGroupedKernel< |
| 130 | + Index1D, ArrayView1D<TValue, Stride1D.Dense>, ArrayView1D<TValue, TValueStride>>( |
| 131 | + WebGLScatterRadixCopyOut<TValue, TValueStride>); |
| 132 | + var extractBit = accelerator.LoadAutoGroupedKernel< |
| 133 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, int>( |
| 134 | + WebGLScatterRadixExtractBitFloat4E2M1<TRadixSortOperation>); |
| 135 | + var computeDest = accelerator.LoadAutoGroupedKernel< |
| 136 | + Index1D, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, |
| 137 | + ArrayView1D<int, Stride1D.Dense>, int>(WebGLScatterRadixComputeDest); |
| 138 | + var exclusiveScan = accelerator.CreateScan< |
| 139 | + int, Stride1D.Dense, Stride1D.Dense, AddInt32>(ScanKind.Exclusive); |
| 140 | + |
| 141 | + int numBits = default(TRadixSortOperation).NumBits; // 4 |
| 142 | + string valType = WebGLScatterValueType<TValue>(); |
| 143 | + int valCpe = WebGLScatterCpe<TValue>(); |
| 144 | + |
| 145 | + return (stream, keys, values, tempView) => |
| 146 | + { |
| 147 | + int n = (int)keys.Length; |
| 148 | + if (n <= 1) |
| 149 | + return; |
| 150 | + |
| 151 | + using var keysA = accelerator.Allocate1D<float>(n); |
| 152 | + using var keysB = accelerator.Allocate1D<float>(n); |
| 153 | + using var valsA = accelerator.Allocate1D<TValue>(n); |
| 154 | + using var valsB = accelerator.Allocate1D<TValue>(n); |
| 155 | + using var flags = accelerator.Allocate1D<int>(n); |
| 156 | + using var onePrefix = accelerator.Allocate1D<int>(n); |
| 157 | + using var dest = accelerator.Allocate1D<int>(n); |
| 158 | + using var scanTemp = accelerator.Allocate1D<int>(1); |
| 159 | + |
| 160 | + copyInKeys(stream, n, keys, keysA.View); |
| 161 | + copyInVals(stream, n, values, valsA.View); |
| 162 | + |
| 163 | + var kSrc = keysA; |
| 164 | + var kDst = keysB; |
| 165 | + var vSrc = valsA; |
| 166 | + var vDst = valsB; |
| 167 | + for (int bit = 0; bit < numBits; bit++) |
| 168 | + { |
| 169 | + extractBit(stream, n, kSrc.View, flags.View, bit); |
| 170 | + exclusiveScan(stream, flags.View, onePrefix.View, scanTemp.View); |
| 171 | + computeDest(stream, n, flags.View, onePrefix.View, dest.View, n); |
| 172 | + scatter.Scatter(kDst.View, kSrc.View, dest.View, n, "float", 1); |
| 173 | + scatter.Scatter(vDst.View, vSrc.View, dest.View, n, valType, valCpe); |
| 174 | + var kt = kSrc; kSrc = kDst; kDst = kt; |
| 175 | + var vt = vSrc; vSrc = vDst; vDst = vt; |
| 176 | + } |
| 177 | + |
| 178 | + copyOutKeys(stream, n, kSrc.View, keys); |
| 179 | + copyOutVals(stream, n, vSrc.View, values); |
| 180 | + }; |
| 181 | + } |
| 182 | + } |
| 183 | +} |
0 commit comments