|
| 1 | +// --------------------------------------------------------------------------------------- |
| 2 | +// ILGPU Algorithms |
| 3 | +// Copyright (c) 2019-2023 ILGPU Project |
| 4 | +// www.ilgpu.net |
| 5 | +// |
| 6 | +// File: RadixSortExtensions.Float8E4M3.cs |
| 7 | +// |
| 8 | +// This file is part of ILGPU and is distributed under the University of Illinois Open |
| 9 | +// Source License. See LICENSE.txt for details. |
| 10 | +// --------------------------------------------------------------------------------------- |
| 11 | + |
| 12 | +using ILGPU.Algorithms.RadixSortOperations; |
| 13 | +using ILGPU.Algorithms.ScanReduceOperations; |
| 14 | +using ILGPU.Runtime; |
| 15 | +using System.Runtime.CompilerServices; |
| 16 | + |
| 17 | +namespace ILGPU.Algorithms |
| 18 | +{ |
| 19 | + // WebGL Float8E4M3-key radix sort. FP8 is a 1-byte sub-word key; the WebGL |
| 20 | + // render-to-texture scatter writes WHOLE 32-bit texels and cannot move a sub-texel |
| 21 | + // value, so - exactly like Half and BFloat16 (RadixSortExtensions.cs / |
| 22 | + // RadixSortExtensions.BFloat16.cs) - FP8 sorts via an UNPACKED f32 working |
| 23 | + // representation: copy-in widens each Float8E4M3 to f32 (lossless: every FP8 value is a |
| 24 | + // strict subset of f32), the radix bit is derived by narrowing back to Float8E4M3 and |
| 25 | + // calling the canonical ExtractRadixBits, and copy-out narrows the sorted f32 back to |
| 26 | + // Float8E4M3 (exact round-trip for any value that began as a Float8E4M3). Mirrors the |
| 27 | + // BFloat16 path one-for-one. |
| 28 | + static partial class RadixSortExtensions |
| 29 | + { |
| 30 | + private static void WebGLScatterRadixCopyInFloat8E4M3<TStride>( |
| 31 | + Index1D index, ArrayView1D<Float8E4M3, TStride> input, |
| 32 | + ArrayView1D<float, Stride1D.Dense> output) |
| 33 | + where TStride : struct, IStride1D => |
| 34 | + output[index.X] = (float)input[index.X]; |
| 35 | + |
| 36 | + private static void WebGLScatterRadixCopyOutFloat8E4M3<TStride>( |
| 37 | + Index1D index, ArrayView1D<float, Stride1D.Dense> input, |
| 38 | + ArrayView1D<Float8E4M3, TStride> output) |
| 39 | + where TStride : struct, IStride1D => |
| 40 | + output[index.X] = (Float8E4M3)input[index.X]; |
| 41 | + |
| 42 | + private static void WebGLScatterRadixExtractBitFloat8E4M3<TRadixSortOperation>( |
| 43 | + Index1D index, ArrayView1D<float, Stride1D.Dense> keys, |
| 44 | + ArrayView1D<int, Stride1D.Dense> flags, int bit) |
| 45 | + where TRadixSortOperation : struct, IRadixSortOperation<Float8E4M3> |
| 46 | + { |
| 47 | + TRadixSortOperation op = default; |
| 48 | + flags[index.X] = op.ExtractRadixBits((Float8E4M3)keys[index.X], bit, 1); |
| 49 | + } |
| 50 | + |
| 51 | + // Keys-only Float8E4M3 sort. Invoked by reflection from CreateRadixSort (the outer |
| 52 | + // method is generic on T; the compiler can't see T == Float8E4M3 to bind the |
| 53 | + // IRadixSortOperation<Float8E4M3> constraint statically). Called once per handler. |
| 54 | + private static RadixSort<Float8E4M3, TStride> CreateWebGLScatterRadixSortFloat8E4M3< |
| 55 | + TStride, TRadixSortOperation>(Accelerator accelerator, IScatterProvider scatter) |
| 56 | + where TStride : struct, IStride1D |
| 57 | + where TRadixSortOperation : struct, IRadixSortOperation<Float8E4M3> |
| 58 | + { |
| 59 | + var copyIn = accelerator.LoadAutoGroupedKernel< |
| 60 | + Index1D, ArrayView1D<Float8E4M3, TStride>, ArrayView1D<float, Stride1D.Dense>>( |
| 61 | + WebGLScatterRadixCopyInFloat8E4M3<TStride>); |
| 62 | + var copyOut = accelerator.LoadAutoGroupedKernel< |
| 63 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<Float8E4M3, TStride>>( |
| 64 | + WebGLScatterRadixCopyOutFloat8E4M3<TStride>); |
| 65 | + var extractBit = accelerator.LoadAutoGroupedKernel< |
| 66 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, int>( |
| 67 | + WebGLScatterRadixExtractBitFloat8E4M3<TRadixSortOperation>); |
| 68 | + var computeDest = accelerator.LoadAutoGroupedKernel< |
| 69 | + Index1D, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, |
| 70 | + ArrayView1D<int, Stride1D.Dense>, int>(WebGLScatterRadixComputeDest); |
| 71 | + var exclusiveScan = accelerator.CreateScan< |
| 72 | + int, Stride1D.Dense, Stride1D.Dense, AddInt32>(ScanKind.Exclusive); |
| 73 | + |
| 74 | + int numBits = default(TRadixSortOperation).NumBits; // 8 |
| 75 | + |
| 76 | + return (stream, view, temp) => |
| 77 | + { |
| 78 | + int n = (int)view.Length; |
| 79 | + if (n <= 1) |
| 80 | + return; |
| 81 | + |
| 82 | + using var keysA = accelerator.Allocate1D<float>(n); |
| 83 | + using var keysB = accelerator.Allocate1D<float>(n); |
| 84 | + using var flags = accelerator.Allocate1D<int>(n); |
| 85 | + using var onePrefix = accelerator.Allocate1D<int>(n); |
| 86 | + using var dest = accelerator.Allocate1D<int>(n); |
| 87 | + using var scanTemp = accelerator.Allocate1D<int>(1); |
| 88 | + |
| 89 | + copyIn(stream, n, view, keysA.View); |
| 90 | + |
| 91 | + var src = keysA; |
| 92 | + var dst = keysB; |
| 93 | + for (int bit = 0; bit < numBits; bit++) |
| 94 | + { |
| 95 | + extractBit(stream, n, src.View, flags.View, bit); |
| 96 | + exclusiveScan(stream, flags.View, onePrefix.View, scanTemp.View); |
| 97 | + computeDest(stream, n, flags.View, onePrefix.View, dest.View, n); |
| 98 | + scatter.Scatter(dst.View, src.View, dest.View, n, "float"); |
| 99 | + var tmp = src; src = dst; dst = tmp; |
| 100 | + } |
| 101 | + |
| 102 | + copyOut(stream, n, src.View, view); |
| 103 | + }; |
| 104 | + } |
| 105 | + |
| 106 | + |
| 107 | + // Float8E4M3-KEY pairs sort (FP8 key + any 4/8-byte non-FP8 value). Keys use the |
| 108 | + // unpacked f32 working representation; values use the same int/float/uint scatter |
| 109 | + // program as the generic pairs path. Invoked by reflection from CreateRadixSortPairs. |
| 110 | + private static RadixSortPairs<Float8E4M3, TKeyStride, TValue, TValueStride> |
| 111 | + CreateWebGLScatterRadixSortPairsFloat8E4M3Key< |
| 112 | + TKeyStride, TValue, TValueStride, TRadixSortOperation>( |
| 113 | + Accelerator accelerator, IScatterProvider scatter) |
| 114 | + where TKeyStride : struct, IStride1D |
| 115 | + where TValue : unmanaged |
| 116 | + where TValueStride : struct, IStride1D |
| 117 | + where TRadixSortOperation : struct, IRadixSortOperation<Float8E4M3> |
| 118 | + { |
| 119 | + var copyInKeys = accelerator.LoadAutoGroupedKernel< |
| 120 | + Index1D, ArrayView1D<Float8E4M3, TKeyStride>, ArrayView1D<float, Stride1D.Dense>>( |
| 121 | + WebGLScatterRadixCopyInFloat8E4M3<TKeyStride>); |
| 122 | + var copyOutKeys = accelerator.LoadAutoGroupedKernel< |
| 123 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<Float8E4M3, TKeyStride>>( |
| 124 | + WebGLScatterRadixCopyOutFloat8E4M3<TKeyStride>); |
| 125 | + var copyInVals = accelerator.LoadAutoGroupedKernel< |
| 126 | + Index1D, ArrayView1D<TValue, TValueStride>, ArrayView1D<TValue, Stride1D.Dense>>( |
| 127 | + WebGLScatterRadixCopyIn<TValue, TValueStride>); |
| 128 | + var copyOutVals = accelerator.LoadAutoGroupedKernel< |
| 129 | + Index1D, ArrayView1D<TValue, Stride1D.Dense>, ArrayView1D<TValue, TValueStride>>( |
| 130 | + WebGLScatterRadixCopyOut<TValue, TValueStride>); |
| 131 | + var extractBit = accelerator.LoadAutoGroupedKernel< |
| 132 | + Index1D, ArrayView1D<float, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, int>( |
| 133 | + WebGLScatterRadixExtractBitFloat8E4M3<TRadixSortOperation>); |
| 134 | + var computeDest = accelerator.LoadAutoGroupedKernel< |
| 135 | + Index1D, ArrayView1D<int, Stride1D.Dense>, ArrayView1D<int, Stride1D.Dense>, |
| 136 | + ArrayView1D<int, Stride1D.Dense>, int>(WebGLScatterRadixComputeDest); |
| 137 | + var exclusiveScan = accelerator.CreateScan< |
| 138 | + int, Stride1D.Dense, Stride1D.Dense, AddInt32>(ScanKind.Exclusive); |
| 139 | + |
| 140 | + int numBits = default(TRadixSortOperation).NumBits; // 8 |
| 141 | + string valType = WebGLScatterValueType<TValue>(); |
| 142 | + int valCpe = WebGLScatterCpe<TValue>(); |
| 143 | + |
| 144 | + return (stream, keys, values, tempView) => |
| 145 | + { |
| 146 | + int n = (int)keys.Length; |
| 147 | + if (n <= 1) |
| 148 | + return; |
| 149 | + |
| 150 | + using var keysA = accelerator.Allocate1D<float>(n); |
| 151 | + using var keysB = accelerator.Allocate1D<float>(n); |
| 152 | + using var valsA = accelerator.Allocate1D<TValue>(n); |
| 153 | + using var valsB = accelerator.Allocate1D<TValue>(n); |
| 154 | + using var flags = accelerator.Allocate1D<int>(n); |
| 155 | + using var onePrefix = accelerator.Allocate1D<int>(n); |
| 156 | + using var dest = accelerator.Allocate1D<int>(n); |
| 157 | + using var scanTemp = accelerator.Allocate1D<int>(1); |
| 158 | + |
| 159 | + copyInKeys(stream, n, keys, keysA.View); |
| 160 | + copyInVals(stream, n, values, valsA.View); |
| 161 | + |
| 162 | + var kSrc = keysA; |
| 163 | + var kDst = keysB; |
| 164 | + var vSrc = valsA; |
| 165 | + var vDst = valsB; |
| 166 | + for (int bit = 0; bit < numBits; bit++) |
| 167 | + { |
| 168 | + extractBit(stream, n, kSrc.View, flags.View, bit); |
| 169 | + exclusiveScan(stream, flags.View, onePrefix.View, scanTemp.View); |
| 170 | + computeDest(stream, n, flags.View, onePrefix.View, dest.View, n); |
| 171 | + scatter.Scatter(kDst.View, kSrc.View, dest.View, n, "float", 1); |
| 172 | + scatter.Scatter(vDst.View, vSrc.View, dest.View, n, valType, valCpe); |
| 173 | + var kt = kSrc; kSrc = kDst; kDst = kt; |
| 174 | + var vt = vSrc; vSrc = vDst; vDst = vt; |
| 175 | + } |
| 176 | + |
| 177 | + copyOutKeys(stream, n, kSrc.View, keys); |
| 178 | + copyOutVals(stream, n, vSrc.View, values); |
| 179 | + }; |
| 180 | + } |
| 181 | + } |
| 182 | +} |
0 commit comments