@@ -109,14 +109,60 @@ function partial_mapreduce_device(f, op, neutral, maxitems, Rreduce, Rother, R,
109109 return
110110end
111111
112+ # Coalesced reduction for when the contiguous leading dimension is NOT reduced (the reduced
113+ # axes are strided). One work-item per output slice (Rother element), grid-strided; each
114+ # serially reduces over Rreduce. Consecutive work-items map to consecutive output slices,
115+ # which are consecutive in memory, so global reads are coalesced across lanes — the access
116+ # pattern a `dims=1` reduction already uses. On the Aurora LTS stack the workgroup-per-slice
117+ # kernel above reads a *strided* reduced dimension non-coalesced, which silently corrupts
118+ # large reductions (e.g. `sum(A; dims=2)`); this path avoids that pattern entirely.
119+ function coalesced_mapreduce_device (f, op, neutral, Rreduce, Rother, R, As... )
120+ iother = (get_group_id () - 1 ) * get_local_size () + get_local_id ()
121+ gstride = get_num_groups () * get_local_size ()
122+ @inbounds while iother <= length (Rother)
123+ Iother = Rother[iother]
124+ Iout = CartesianIndex (Tuple (Iother)... , 1 )
125+ neut = neutral === nothing ? R[Iout] : neutral
126+ val = op (neut, neut)
127+ for ireduce in 1 : length (Rreduce)
128+ Ireduce = Rreduce[ireduce]
129+ J = max (Iother, Ireduce)
130+ val = op (val, f (_map_getindex (As, J)... ))
131+ end
132+ R[Iout] = val
133+ iother += gstride
134+ end
135+ return
136+ end
137+
112138# # COV_EXCL_STOP
113139
140+ # Aurora LTS workaround: the NEO/IGC LTS stack miscompiles *strided* (non-coalesced) global
141+ # reads inside the reduction kernel, silently corrupting results whenever an input is read
142+ # along a non-contiguous axis (e.g. `a == transpose(b)`, `sum(transpose(x))`, `ishermitian`).
143+ # Elementwise copies are NOT affected. `_dense_reduce_input` returns false for any input that
144+ # reads non-contiguous memory (a transposed/permuted/strided view, or a broadcast containing
145+ # one), so such inputs get materialized to a dense `oneArray` before reducing. See
146+ # ISSUE_mapreduce_corruption.md and the `naive_transpose` (`a == transpose(b)`) failure.
147+ @inline _dense_reduce_input (:: oneArray ) = true
148+ @inline _dense_reduce_input (x:: Base.ReshapedArray ) = _dense_reduce_input (parent (x))
149+ @inline _dense_reduce_input (:: AbstractArray ) = false # Transpose/Adjoint/PermutedDims/SubArray/…
150+ @inline _dense_reduce_input (:: Any ) = true # scalars/Refs/tuples carried in a broadcast
151+ @inline _dense_reduce_input (bc:: Broadcast.Broadcasted ) = all (_dense_reduce_input, bc. args)
152+
114153function GPUArrays. mapreducedim! (f:: F , op:: OP , R:: oneWrappedArray{T} ,
115154 A:: Union{AbstractArray,Broadcast.Broadcasted} ;
116155 init= nothing ) where {F, OP, T}
117156 Base. check_reducedims (R, A)
118157 length (A) == 0 && return R # isempty(::Broadcasted) iterates
119158
159+ # Aurora LTS workaround (see `_dense_reduce_input` above): materialize strided inputs to a
160+ # dense array first so every global read in the reduction kernel is coalesced.
161+ if ! _dense_reduce_input (A)
162+ Acontig = Broadcast. materialize (Broadcast. broadcasted (f, A))
163+ return GPUArrays. mapreducedim! (identity, op, R, Acontig; init= init)
164+ end
165+
120166 # add singleton dimensions to the output container, if needed
121167 if ndims (R) < ndims (A)
122168 dims = Base. fill_to_length (size (R), 1 , Val (ndims (A)))
@@ -137,6 +183,20 @@ function GPUArrays.mapreducedim!(f::F, op::OP, R::oneWrappedArray{T},
137183 # but allows us to write a generalized kernel supporting partial reductions.
138184 R′ = reshape (R, (size (R)... , 1 ))
139185
186+ # Aurora LTS workaround: the workgroup-per-slice kernel below reads a *strided* reduced
187+ # dimension non-coalesced, which silently corrupts reductions on this stack (regardless of
188+ # output count — it depends on the reduction length, not the number of slices). Whenever
189+ # the contiguous leading dimension is NOT reduced (`size(Rreduce, 1) == 1`), use the
190+ # coalesced one-work-item-per-slice kernel, whose lanes read consecutive memory. Few-slice
191+ # reductions get less parallelism but stay correct; the common many-slice case is also fast.
192+ if size (Rreduce, 1 ) == 1
193+ items = clamp (length (Rother), 1 , 256 )
194+ groups = min (cld (length (Rother), items), 1024 )
195+ @oneapi items= items groups= groups coalesced_mapreduce_device (
196+ f, op, init, Rreduce, Rother, R′, A)
197+ return R
198+ end
199+
140200 # how many items do we want?
141201 #
142202 # items in a group work together to reduce values across the reduction dimensions;
0 commit comments