Gate the LTS workarounds behind a runtime ONEAPI_LTS flag (layers 1-2)

Make the Intel LTS-stack support opt-in so this branch can eventually merge
into main without affecting the rolling stack: with the flag off, every LTS
code path is bypassed and behavior matches upstream.

Introduce oneL0.LTS[], resolved at the top of oneL0.__init__ from ONEAPI_LTS
(default on for this branch; set ONEAPI_LTS=0 for the rolling-stack paths).
Resolved before the driver-availability early returns so it is set even on a
host without a functional GPU. Gate on it:

  - Layer 1 (behavior): the strided-reduction materialization and the
    coalesced-kernel dispatch in mapreducedim!; the command-queue registry
    (global_queue / synchronize_all_queues), the pre-free synchronize in
    release, and the queue-finalizer drain + handle-null in cmdqueue.jl.
  - Layer 2 (codegen): _compiler_config now selects the SPIRVCompilerTarget
    backend (:khronos translator vs :llvm back-end) and the SPIR-V extension
    list from the flag. GPUCompiler loads the tool lazily, so both SPIR-V JLLs
    can coexist as deps and the choice is made at compile time.

Deferred to layer 3: driver-JLL selection (NEO/loader/headers/igc) via
Preferences and the oneMKL support-library ABI. The oneMKL FFT GC-rooting is
left as-is (already fixed upstream via GC.@preserve).

Verified: precompiles and loads; ONEAPI_LTS unset/1 -> on, ONEAPI_LTS=0 -> off.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: michel2323

lib/level-zero/cmdqueue.jl

@@ -20,17 +20,21 @@ mutable struct ZeCommandQueue
         zeCommandQueueCreate(ctx, dev, desc_ref, handle_ref)
         obj = new(handle_ref[], ctx, dev, ordinal)
         finalizer(obj) do obj
-            # the queue may still have work in flight (nothing requires a task to
-            # synchronize before dying), and zeCommandQueueDestroy does not wait for
-            # it: on the LTS NEO stack the still-running work then faults as soon as
-            # a referenced allocation is freed, getting the context banned. drain the
-            # queue first; unchecked, as sync on a banned context returns an error.
-            unchecked_zeCommandQueueSynchronize(obj, typemax(UInt64))
+            if LTS[]
+                # the queue may still have work in flight (nothing requires a task to
+                # synchronize before dying), and zeCommandQueueDestroy does not wait for
+                # it: on the LTS NEO stack the still-running work then faults as soon as
+                # a referenced allocation is freed, getting the context banned. drain the
+                # queue first; unchecked, as sync on a banned context returns an error.
+                unchecked_zeCommandQueueSynchronize(obj, typemax(UInt64))
+            end
             zeCommandQueueDestroy(obj)
-            # mark the queue as destroyed: it can still be weakly reachable (e.g. from
-            # the queue registry used by `synchronize_all_queues`), and synchronizing a
-            # destroyed handle crashes in the driver.
-            obj.handle = ze_command_queue_handle_t(C_NULL)
+            if LTS[]
+                # mark the queue as destroyed: it can still be weakly reachable (e.g. from
+                # the queue registry used by `synchronize_all_queues`), and synchronizing a
+                # destroyed handle crashes in the driver.
+                obj.handle = ze_command_queue_handle_t(C_NULL)
+            end
         end
         obj
     end

lib/level-zero/oneL0.jl

@@ -139,10 +139,26 @@ const functional = Ref{Bool}(false)
 const validation_layer = Ref{Bool}()
 const parameter_validation = Ref{Bool}()
 
+# Master switch for the Intel LTS-stack workarounds (driver/IGC quirks on the Aurora
+# LTS NEO 25.18 stack): the SPIR-V translator codegen path, strided-reduction
+# materialization, and command-queue drain-before-free. All such code paths are gated
+# on `LTS[]`, so with it disabled the package behaves exactly like the upstream rolling
+# stack. Default on for this branch (which pins the LTS toolchain); set `ONEAPI_LTS=0`
+# to exercise the rolling-stack paths. When merged upstream the default flips to false
+# and an LTS deployment opts in with `ONEAPI_LTS=1`.
+const LTS = Ref{Bool}(true)
+
 function __init__()
     precompiling = ccall(:jl_generating_output, Cint, ()) != 0
     precompiling && return
 
+    # Resolve the LTS master switch up front, before the driver-availability early
+    # returns below: it gates codegen and behavior and must be set even on hosts
+    # without a functional GPU. Default on for this (LTS) branch; flip the "true"
+    # default to "false" when merged onto the rolling stack. ONEAPI_LTS=0 forces the
+    # rolling-stack code paths.
+    LTS[] = lowercase(get(ENV, "ONEAPI_LTS", "true")) in ("1", "true", "yes", "on")
+
     if Sys.iswindows()
         if Libdl.dlopen(libze_loader; throw_error=false) === nothing
             @error "The oneAPI Level Zero loader was not found. Please ensure the Intel GPU drivers are installed."

src/compiler/compilation.jl

@@ -187,14 +187,25 @@ end
     supports_fp16 = oneL0.module_properties(device()).fp16flags & oneL0.ZE_DEVICE_MODULE_FLAG_FP16 == oneL0.ZE_DEVICE_MODULE_FLAG_FP16
     supports_fp64 = oneL0.module_properties(device()).fp64flags & oneL0.ZE_DEVICE_MODULE_FLAG_FP64 == oneL0.ZE_DEVICE_MODULE_FLAG_FP64
 
-    # TODO: emit printf format strings in constant memory
-    extensions = String[
-        "SPV_EXT_relaxed_printf_string_address_space",
-        "SPV_EXT_shader_atomic_float_add"
-    ]
+    # SPIR-V codegen path. The Aurora LTS NEO/IGC runtime only accepts SPIR-V from the
+    # Khronos translator and needs these extensions declared explicitly; the rolling stack
+    # uses the LLVM SPIR-V back-end (which handles the extensions itself). GPUCompiler picks
+    # the tool from the target's `backend` field and loads the JLL lazily, so both can be
+    # listed as deps and the choice is made here at compile time.
+    if oneL0.LTS[]
+        backend = :khronos
+        # TODO: emit printf format strings in constant memory
+        extensions = String[
+            "SPV_EXT_relaxed_printf_string_address_space",
+            "SPV_EXT_shader_atomic_float_add"
+        ]
+    else
+        backend = :llvm
+        extensions = String[]
+    end
 
     # create GPUCompiler objects
-    target = SPIRVCompilerTarget(; extensions, supports_fp16, supports_fp64, kwargs...)
+    target = SPIRVCompilerTarget(; backend, extensions, supports_fp16, supports_fp64, kwargs...)
     params = oneAPICompilerParams()
     CompilerConfig(target, params; kernel, name, always_inline)
 end

src/context.jl

@@ -225,17 +225,19 @@ function global_queue(ctx::ZeContext, dev::ZeDevice)
     #       objects should track ownership, and not rely on implicit global state.
     get!(task_local_storage(), (:ZeCommandQueue, ctx, dev)) do
         queue = ZeCommandQueue(ctx, dev; flags = oneL0.ZE_COMMAND_QUEUE_FLAG_IN_ORDER)
-        # disable finalizers while mutating the registry: a GC-driven finalizer on this
-        # task could call back into `synchronize_all_queues` (the lock is reentrant) and
-        # observe/mutate the registry mid-update.
-        GC.enable_finalizers(false)
-        try
-            @lock queue_registry_lock begin
-                push!(get!(Vector{Tuple{WeakRef,ZeCommandQueue}}, queue_registry, (ctx, dev)),
-                      (WeakRef(current_task()), queue))
+        if oneL0.LTS[]
+            # disable finalizers while mutating the registry: a GC-driven finalizer on this
+            # task could call back into `synchronize_all_queues` (the lock is reentrant) and
+            # observe/mutate the registry mid-update.
+            GC.enable_finalizers(false)
+            try
+                @lock queue_registry_lock begin
+                    push!(get!(Vector{Tuple{WeakRef,ZeCommandQueue}}, queue_registry, (ctx, dev)),
+                          (WeakRef(current_task()), queue))
+                end
+            finally
+                GC.enable_finalizers(true)
             end
-        finally
-            GC.enable_finalizers(true)
         end
         queue
     end
@@ -259,6 +261,9 @@ const queue_registry = Dict{Tuple{ZeContext,ZeDevice},Vector{Tuple{WeakRef,ZeCom
 # i.e., all queues whose in-flight work could possibly reference an allocation that is
 # about to be freed.
 function synchronize_all_queues(ctx::ZeContext, dev::Union{ZeDevice,Nothing})
+    # only the LTS stack populates the queue registry (see `global_queue`); on the
+    # rolling stack this is a no-op and `release` frees directly.
+    oneL0.LTS[] || return
     queues = ZeCommandQueue[]
     stale = Tuple{WeakRef,ZeCommandQueue}[]
     GC.enable_finalizers(false)

src/mapreduce.jl

@@ -158,7 +158,7 @@ function GPUArrays.mapreducedim!(f::F, op::OP, R::oneWrappedArray{T},
 
     # Aurora LTS workaround (see `_dense_reduce_input` above): materialize strided inputs to a
     # dense array first so every global read in the reduction kernel is coalesced.
-    if !_dense_reduce_input(A)
+    if oneL0.LTS[] && !_dense_reduce_input(A)
         Acontig = Broadcast.materialize(Broadcast.broadcasted(f, A))
         return GPUArrays.mapreducedim!(identity, op, R, Acontig; init=init)
     end
@@ -189,7 +189,7 @@ function GPUArrays.mapreducedim!(f::F, op::OP, R::oneWrappedArray{T},
     # the contiguous leading dimension is NOT reduced (`size(Rreduce, 1) == 1`), use the
     # coalesced one-work-item-per-slice kernel, whose lanes read consecutive memory. Few-slice
     # reductions get less parallelism but stay correct; the common many-slice case is also fast.
-    if size(Rreduce, 1) == 1
+    if oneL0.LTS[] && size(Rreduce, 1) == 1
         items = clamp(length(Rother), 1, 256)
         groups = min(cld(length(Rother), items), 1024)
         @oneapi items=items groups=groups coalesced_mapreduce_device(

src/pool.jl

@@ -89,8 +89,11 @@ function release(buf::oneL0.AbstractBuffer)
     # That turns a GC-driven free of a dead array whose last kernel/copy hasn't retired
     # into a GPU pagefault, which gets the kernel context banned and makes every later
     # submission fail with ZE_RESULT_ERROR_UNKNOWN. Synchronize the queues that could
-    # reference this buffer before freeing.
-    synchronize_all_queues(oneL0.context(buf), oneL0.device(buf))
+    # reference this buffer before freeing. (No-op on the rolling stack, which honors
+    # BLOCKING_FREE.)
+    if oneL0.LTS[]
+        synchronize_all_queues(oneL0.context(buf), oneL0.device(buf))
+    end
 
     free(buf; policy=oneL0.ZE_DRIVER_MEMORY_FREE_POLICY_EXT_FLAG_BLOCKING_FREE)