From c33b0f5bbfd139f75e00ae3eee871804f487bfba Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Wed, 15 Jul 2026 18:39:36 +0200 Subject: [PATCH 01/45] Make GPUCompiler IR relocatable across Julia sessions --- src/driver.jl | 30 +++---- src/interface.jl | 78 ++++++++++++++++++ src/irgen.jl | 29 +++++-- src/jlgen.jl | 204 ++++++++++++++++++++++++++++++++++++---------- src/mcgen.jl | 70 +++++++++++----- src/optim.jl | 25 +++--- src/rtlib.jl | 36 ++++---- src/validation.jl | 72 +++++++++------- test/native.jl | 140 +++++++++++++++++++++++++++---- test/ptx.jl | 21 ++++- 10 files changed, 548 insertions(+), 157 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index 38581053..c5a932a3 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -93,7 +93,7 @@ function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) else error("Unknown assembly format $output") end - asm, asm_meta = emit_asm(job, ir, format) + asm, asm_meta = emit_asm(job, ir, ir_meta.host_references, format) if output == :asm || output == :obj return asm, (; asm_meta..., ir_meta..., ir) @@ -192,6 +192,7 @@ const __llvm_initialized = Ref(false) # finalize the current module. this needs to happen before linking deferred modules, # since those modules have been finalized themselves, and we don't want to re-finalize. entry = finish_module!(job, ir, entry) + host_references = classify_gvs!(ir, gv_to_value) # deferred code generation has_deferred_jobs = job.config.toplevel && !job.config.only_entry && @@ -245,11 +246,9 @@ const __llvm_initialized = Ref(false) dyn_ir, dyn_meta = @invokelatest deferred_codegen(dyn_job, job) dyn_entry_fn = LLVM.name(dyn_meta.entry) merge!(compiled, dyn_meta.compiled) - if haskey(dyn_meta, :gv_to_value) - merge!(gv_to_value, dyn_meta.gv_to_value) - end @assert context(dyn_ir) == context(ir) - link!(ir, dyn_ir) + link_with_host_references!(ir, host_references, dyn_ir, + dyn_meta.host_references) changed = true dyn_entry_fn end @@ -292,7 +291,7 @@ const __llvm_initialized = Ref(false) if job.config.toplevel && job.config.libraries # load the runtime outside of a timing block (because it recurses into the compiler) if !uses_julia_runtime(job) - runtime = load_runtime(job) + runtime, runtime_references = load_runtime(job) end @tracepoint "Library linking" begin @@ -301,7 +300,8 @@ const __llvm_initialized = Ref(false) # GPU run-time library if !uses_julia_runtime(job) - @tracepoint "runtime library" link!(ir, runtime; only_needed=true) + @tracepoint "runtime library" link_with_host_references!( + ir, host_references, runtime, runtime_references; only_needed=true) end end end @@ -334,13 +334,12 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) - # Materialize isbits and Bool boxes; bake addresses for other objects. - portable = relocate_gvs!(ir, gv_to_value) - portable || mark_session_dependent!(job) + host_references.embedded_pointer |= !materialize_bool_singletons!(ir) + host_references.embedded_pointer && mark_session_dependent!(job) if job.config.optimize @tracepoint "optimization" begin - optimize!(job, ir; job.config.opt_level) + optimize!(job, ir, host_references; job.config.opt_level) # deferred codegen has some special optimization requirements, # which also need to happen _after_ regular optimization. @@ -405,7 +404,7 @@ const __llvm_initialized = Ref(false) if job.config.toplevel && job.config.validate @tracepoint "validation" begin - check_ir(job, ir) + check_ir(job, ir, host_references) end end @@ -413,18 +412,19 @@ const __llvm_initialized = Ref(false) @tracepoint "verification" verify(ir) end - return ir, (; entry, compiled, gv_to_value) + return ir, (; entry, compiled, host_references) end @locked function emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, - format::LLVM.API.LLVMCodeGenFileType) + refs::HostReferences, format::LLVM.API.LLVMCodeGenFileType) # NOTE: strip after validation to get better errors if job.config.strip @tracepoint "Debug info removal" strip_debuginfo!(ir) end @tracepoint "LLVM back-end" begin - @tracepoint "preparation" prepare_execution!(job, ir) + @tracepoint "preparation" prepare_execution!(job, ir, refs) + refs.embedded_pointer && mark_session_dependent!(job) code = @tracepoint "machine-code generation" mcgen(job, ir, format) end diff --git a/src/interface.jl b/src/interface.jl index f17c6e3c..4253c6b4 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -429,6 +429,82 @@ the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock """ function cached_results end +""" + JuliaValueRef(value) + +A Julia value used as the serializable identity of a host reference. Resolve it in the active +session with [`resolve_host_reference`](@ref); a loader that writes the resulting address into +device storage must keep `value` rooted for as long as that storage remains reachable. +""" +struct JuliaValueRef + value::Any +end + +""" + CGlobalRef(symbol) + +A libjulia C data global. Resolving this reference calls `jl_cglobal` for `symbol` and loads +the word stored at that address. +""" +struct CGlobalRef + symbol::Symbol +end + +""" + HostReference + +A serializable source for a host-derived word: either a [`JuliaValueRef`](@ref) or a +[`CGlobalRef`](@ref). +""" +const HostReference = Union{JuliaValueRef,CGlobalRef} + +""" + HostReferences(slots, embedded_pointer) + +Host-reference metadata accompanying a module. `slots` maps each object-code symbol to the +source of the word that must be written there. `embedded_pointer` conservatively records that +an unavoidable session pointer has been embedded, so the artifact cannot be serialized into a +package image even if `slots` is empty. +""" +mutable struct HostReferences + slots::Dict{String,HostReference} + embedded_pointer::Bool +end + +HostReferences() = HostReferences(Dict{String,HostReference}(), false) +copy_host_references(refs::HostReferences) = + HostReferences(copy(refs.slots), refs.embedded_pointer) + +same_host_reference(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value +same_host_reference(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol +same_host_reference(::HostReference, ::HostReference) = false + +""" + resolve_host_reference(ref) -> UInt + +Resolve `ref` to its current-session word. Loader-based backends use this after loading the +object and before exposing a callable function. +""" +resolve_host_reference(ref::JuliaValueRef) = + UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), ref.value)) +function resolve_host_reference(ref::CGlobalRef) + address = ccall(:jl_cglobal, Any, (Any, Any), String(ref.symbol), UInt) + return unsafe_load(address) +end + +""" + lower_host_references!(job, mod, refs) + +Lower compiler-managed Julia value globals and Julia runtime globals for `job`'s final +backend representation. The default lowering resolves every live reference in the current +Julia session. Backends with a loader relocation mechanism may call +[`emit_host_reference_slots!`](@ref) instead, then resolve and patch those slots after loading. +""" +function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences) + resolve_host_reference_slots!(mod, refs) +end + @static if HAS_INTEGRATED_CACHE """ @@ -550,6 +626,8 @@ end end # HAS_INTEGRATED_CACHE @public GPUCompilerCacheToken, cache_owner, cached_results +@public JuliaValueRef, CGlobalRef, HostReference, HostReferences +@public lower_host_references!, emit_host_reference_slots!, resolve_host_reference # the method table to use # diff --git a/src/irgen.jl b/src/irgen.jl index 9129e20c..f7056d68 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -43,21 +43,36 @@ function irgen(@nospecialize(job::CompilerJob)) end end - # sanitize global values (Julia doesn't when using the external codegen policy) - for val in [collect(globals(mod)); collect(functions(mod))] - isdeclaration(val) && continue + # Julia value globals can be declarations. Sanitize every definition as well as every + # mapped global, but leave unrelated external declarations alone because their names are + # part of their ABI. + for val in collect(globals(mod)) old_name = LLVM.name(val) + mapped = haskey(gv_to_value, old_name) + isdeclaration(val) && !mapped && continue new_name = safe_name(old_name) if old_name != new_name LLVM.name!(val, new_name) - val = get(gv_to_value, old_name, nothing) - if val !== nothing - delete!(gv_to_value, old_name) - gv_to_value[new_name] = val + actual_name = LLVM.name(val) + if mapped + init = pop!(gv_to_value, old_name) + haskey(gv_to_value, actual_name) && + error("Duplicate Julia value global '$actual_name'") + gv_to_value[actual_name] = init end end end + # sanitize defined functions (Julia doesn't when using the external codegen policy) + for val in collect(functions(mod)) + isdeclaration(val) && continue + old_name = LLVM.name(val) + new_name = safe_name(old_name) + if old_name != new_name + LLVM.name!(val, new_name) + end + end + # rename and process the entry point if job.config.name !== nothing LLVM.name!(entry, safe_name(job.config.name)) diff --git a/src/jlgen.jl b/src/jlgen.jl index d4820377..8c295630 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -756,63 +756,179 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end -""" - relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - -Resolve globals that refer to Julia objects. `jl_true`/`jl_false`, which are -absent from `gv_to_value`, become canonical module-local boxes. Ordinary -non-ghost isbits objects are also materialized; other objects keep their host -address as an opaque identity token. - -Only GVs that are declarations (as produced by `compile_method_instance`, -which strips the initializers for session portability) or still have a null -initializer are touched: preserving existing initializers covers the -older-Julia path where Julia itself emits pointer values directly. - -Apart from the dedicated Bool globals, GVs present in `mod` but missing from -`gv_to_value` remain declarations, which back-ends will reject loudly -(undefined symbol) rather than silently folding to null. - -Returns `true` if the module is session-portable afterwards: no absolute host -address was written (neither a baked slot nor a materialized header carrying -a non-smalltag type pointer). -""" -function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - portable = materialize_bool_singletons!(mod) +function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + refs = HostReferences() mod_gvs = globals(mod) for (name, init) in gv_to_value - # Bools are resolved by name above. - name in ("jl_true", "jl_false") && continue - haskey(mod_gvs, name) || continue gv = mod_gvs[name] cur = initializer(gv) if !(cur === nothing || LLVM.isnull(cur)) - # pre-baked by Julia itself (pre-1.13): also session-absolute - portable = false + refs.embedded_pointer = true continue end - val = nothing - if init != C_NULL - obj = Base.unsafe_pointer_to_objref(init) - # Zero-sized objects remain identity tokens. - if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) - val, hdr = materialize_box!(mod, gv, obj, init) - # non-smalltag headers carry a host DataType pointer - portable &= hdr < UInt(64 << 4) # jl_max_tags << 4 - end - end - if val === nothing - val = const_inttoptr(ConstantInt(Int64(init)), global_value_type(gv)) - portable = false + init == C_NULL && error("Missing Julia object for global '$name'") + obj = Base.unsafe_pointer_to_objref(init) + if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) + val, hdr = materialize_box!(mod, gv, obj, init) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + refs.embedded_pointer |= hdr >= UInt(64 << 4) + else + host_reference_slot_size(mod, gv, name) + refs.slots[name] = JuliaValueRef(obj) end + end + return refs +end + +host_reference_word_type() = LLVM.IntType(8sizeof(UInt)) + +function host_reference_slot_size(mod::LLVM.Module, gv::GlobalVariable, name::String) + size = abi_size(datalayout(mod), global_value_type(gv)) + size == sizeof(UInt) || + error("Host reference slot '$name' has size $size, expected $(sizeof(UInt))") + return +end + +function host_reference_slot_initializer(gv::GlobalVariable, value::UInt) + T = global_value_type(gv) + if T isa LLVM.PointerType + return const_inttoptr(ConstantInt(UInt64(value)), T) + elseif T isa LLVM.IntegerType && width(T) == 8sizeof(UInt) + return ConstantInt(T, value) + end + error("Host reference slot '$(LLVM.name(gv))' has unsupported LLVM type $T") +end + +function check_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + mod_gvs = globals(mod) + for name in keys(refs.slots) + haskey(mod_gvs, name) || error("Missing host reference slot '$name'") + end + return +end + +function prune_dead_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + mod_gvs = globals(mod) + for name in collect(keys(refs.slots)) + haskey(mod_gvs, name) || delete!(refs.slots, name) + end + return +end + +function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + check_host_reference_slots!(mod, refs) + mod_gvs = globals(mod) + for (name, ref) in refs.slots + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + value = resolve_host_reference(ref) + val = host_reference_slot_initializer(gv, value) initializer!(gv, val) - # re-internalize what compile_method_instance demoted to an external - # declaration; with the value in place, the optimizer can now fold it linkage!(gv, LLVM.API.LLVMPrivateLinkage) + refs.embedded_pointer = true end - return portable + empty!(refs.slots) + return +end + +""" + emit_host_reference_slots!(mod, refs) + +Prepare the globals named by `refs.slots` for loader-based lowering. Each remains a writable, +word-sized external symbol that the loader must resolve and patch after loading the object. +""" +function emit_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + check_host_reference_slots!(mod, refs) + mod_gvs = globals(mod) + slots = GlobalVariable[] + for name in keys(refs.slots) + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + initializer!(gv, null(global_value_type(gv))) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, true) + push!(slots, gv) + end + isempty(slots) || set_used!(mod, slots...) + return +end + +function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, + src_mod::LLVM.Module, src_refs::HostReferences; + only_needed=false) + # Link-time optimization may already have removed an unreferenced global. This is the + # last point where its absence is provably dead rather than a broken relocation. + prune_dead_host_reference_slots!(dest_mod, dest_refs) + prune_dead_host_reference_slots!(src_mod, src_refs) + src_gvs = globals(src_mod) + dest_names = Set{String}(LLVM.name(v) for v in [collect(globals(dest_mod)); + collect(functions(dest_mod))]) + src_names = Set{String}(LLVM.name(v) for v in [collect(globals(src_mod)); + collect(functions(src_mod))]) + + function fresh_slot_name(name) + base = safe_name(name) * "_gpucompiler" + candidate = base + suffix = 0 + while candidate in dest_names || candidate in src_names + suffix += 1 + candidate = base * "_" * string(suffix) + end + return candidate + end + + for (name, ref) in collect(src_refs.slots) + dest_ref = get(dest_refs.slots, name, nothing) + haskey(src_gvs, name) || error("Missing source host reference slot '$name'") + if dest_ref !== nothing && same_host_reference(dest_ref, ref) + haskey(globals(dest_mod), name) || + error("Missing destination host reference slot '$name'") + continue + end + + gv = src_gvs[name] + if name in dest_names || dest_ref !== nothing + delete!(src_names, name) + LLVM.name!(gv, fresh_slot_name(name)) + push!(src_names, LLVM.name(gv)) + else + continue + end + actual_name = LLVM.name(gv) + delete!(src_refs.slots, name) + haskey(src_refs.slots, actual_name) && + error("Duplicate source host reference slot '$actual_name'") + src_refs.slots[actual_name] = ref + end + + link!(dest_mod, src_mod; only_needed) + for (name, ref) in src_refs.slots + if !haskey(globals(dest_mod), name) + only_needed && continue + error("Linked host reference slot '$name' is missing") + end + dest_ref = get(dest_refs.slots, name, nothing) + dest_ref === nothing && (dest_refs.slots[name] = ref) + dest_ref === nothing || same_host_reference(dest_ref, ref) || + error("Conflicting linked host reference slot '$name'") + end + for name in keys(dest_refs.slots) + haskey(globals(dest_mod), name) || + error("Merged host reference slot '$name' is missing") + end + dest_refs.embedded_pointer |= src_refs.embedded_pointer + return +end + +function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + refs = classify_gvs!(mod, gv_to_value) + refs.embedded_pointer |= !materialize_bool_singletons!(mod) + resolve_host_reference_slots!(mod, refs) + return !refs.embedded_pointer end # Bool JuliaVariables are absent from `gv_to_value`; define one device box per name. diff --git a/src/mcgen.jl b/src/mcgen.jl index 3fb4a5c6..b7a2438e 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -2,19 +2,25 @@ # final preparations for the module to be compiled to machine code # these passes should not be run when e.g. compiling to write to disk. -function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module) +function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences=HostReferences()) + prune_dead_host_reference_slots!(mod, refs) + collect_runtime_global_references!(job, mod, refs) + lower_host_references!(job, mod, refs) @dispose pb=NewPMPassBuilder() begin - register!(pb, ResolveCPUReferencesPass(job)) + register!(pb, CollectRuntimeGlobalReferencesPass(job, refs)) add!(pb, RecomputeGlobalsAAPass()) add!(pb, GlobalOptPass()) - add!(pb, ResolveCPUReferencesPass(job)) + add!(pb, CollectRuntimeGlobalReferencesPass(job, refs)) add!(pb, GlobalDCEPass()) add!(pb, StripDeadPrototypesPass()) run!(pb, mod, llvm_machine(job.config.target)) end + prune_dead_host_reference_slots!(mod, refs) + lower_host_references!(job, mod, refs) return end @@ -26,25 +32,34 @@ end # but at the same time the GPU can't resolve them at run-time. # # this pass performs that resolution at link time. -struct ResolveCPUReferences +struct CollectRuntimeGlobalReferences job::CompilerJob + refs::HostReferences end -function (self::ResolveCPUReferences)(mod::LLVM.Module) +function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) changed = false - for f in functions(mod) + for f in [collect(functions(mod)); collect(globals(mod))] fn = LLVM.name(f) - if isdeclaration(f) && !LLVM.isintrinsic(f) && startswith(fn, "jl_") - # lazily resolve the address of the binding; some symbols only exist - # within the JIT (e.g. `jl_get_pgcstack_resolved`) and cannot be looked up, - # but such symbols are only ever called, not loaded from. - dereferenced = nothing - function resolve_binding() - if dereferenced === nothing - address = ccall(:jl_cglobal, Any, (Any, Any), fn, UInt) - dereferenced = LLVM.ConstantInt(unsafe_load(address)) + # Julia value globals are already represented by an identity in `refs`; they may + # use a `jl_*` spelling but are not exported libjulia runtime globals. + f isa LLVM.GlobalVariable && haskey(self.refs.slots, fn) && continue + if isdeclaration(f) && (!(f isa LLVM.Function) || !LLVM.isintrinsic(f)) && + startswith(fn, "jl_") + slot = nothing + function runtime_global_slot() + if slot === nothing + name = safe_name("gpu_" * fn) + slot = GlobalVariable(mod, host_reference_word_type(), name) + initializer!(slot, null(global_value_type(slot))) + linkage!(slot, LLVM.API.LLVMExternalLinkage) + extinit!(slot, true) + actual_name = LLVM.name(slot) + haskey(self.refs.slots, actual_name) && + error("Duplicate Julia runtime global slot '$actual_name'") + self.refs.slots[actual_name] = CGlobalRef(Symbol(fn)) end - dereferenced + slot end function replace_bindings!(value) @@ -55,8 +70,20 @@ function (self::ResolveCPUReferences)(mod::LLVM.Module) # recurse changed |= replace_bindings!(val) elseif isa(val, LLVM.LoadInst) - # resolve - replace_uses!(val, resolve_binding()) + T = value_type(val) + if !(T isa LLVM.PointerType || + (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) + error("Unsupported Julia runtime global '$fn' load of LLVM type $T") + end + @dispose builder=IRBuilder() begin + position!(builder, val) + replacement = load!(builder, host_reference_word_type(), + runtime_global_slot()) + if T isa LLVM.PointerType + replacement = inttoptr!(builder, replacement, T) + end + replace_uses!(val, replacement) + end erase!(val) # FIXME: iterator invalidation? changed = true @@ -71,8 +98,11 @@ function (self::ResolveCPUReferences)(mod::LLVM.Module) return changed end -ResolveCPUReferencesPass(job) = - NewPMModulePass("ResolveCPUReferences", ResolveCPUReferences(job)) +collect_runtime_global_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences) = + CollectRuntimeGlobalReferences(job, refs)(mod) +CollectRuntimeGlobalReferencesPass(job, refs=HostReferences()) = + NewPMModulePass("CollectRuntimeGlobalReferences", CollectRuntimeGlobalReferences(job, refs)) function mcgen(@nospecialize(job::CompilerJob), mod::LLVM.Module, format=LLVM.API.LLVMAssemblyFile) diff --git a/src/optim.jl b/src/optim.jl index b0b339a3..de35280a 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -49,7 +49,8 @@ function aggressiveinstcombine_pass(@nospecialize(job::CompilerJob)) end end -function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module; opt_level=2) +function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences; opt_level=2) tm = llvm_machine(job.config.target) tti = llvm_targetinfo(job.config.target) @@ -59,7 +60,7 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module; opt_level= register!(pb, GPULowerCPUFeaturesPass(job)) register!(pb, GPULowerPTLSPass(job)) register!(pb, GPULowerGCFramePass(job)) - register!(pb, GPULinkRuntimePass(job)) + register!(pb, GPULinkRuntimePass(job, refs)) register!(pb, GPULinkLibrariesPass(job)) register!(pb, GPUFinishRuntimeIntrinsicsPass(job)) register!(pb, AddKernelStatePass(job)) @@ -67,7 +68,7 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module; opt_level= register!(pb, CleanupKernelStatePass(job)) add!(pb, NewPMModulePassManager()) do mpm - buildNewPMPipeline!(mpm, job, opt_level) + buildNewPMPipeline!(mpm, job, refs, opt_level) end run!(pb, mod, tm) end @@ -76,7 +77,8 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module; opt_level= return end -function buildNewPMPipeline!(mpm, @nospecialize(job::CompilerJob), opt_level) +function buildNewPMPipeline!(mpm, @nospecialize(job::CompilerJob), refs::HostReferences, + opt_level) buildEarlySimplificationPipeline(mpm, job, opt_level) add!(mpm, AlwaysInlinerPass()) buildEarlyOptimizerPipeline(mpm, job, opt_level) @@ -90,7 +92,7 @@ function buildNewPMPipeline!(mpm, @nospecialize(job::CompilerJob), opt_level) add!(fpm, WarnMissedTransformationsPass()) end end - buildIntrinsicLoweringPipeline(mpm, job, opt_level) + buildIntrinsicLoweringPipeline(mpm, job, refs, opt_level) buildCleanupPipeline(mpm, job, opt_level) end @@ -316,7 +318,8 @@ function buildVectorPipeline(fpm, @nospecialize(job::CompilerJob), opt_level) add!(fpm, InstSimplifyPass()) end -function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), opt_level) +function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), + refs::HostReferences, opt_level) add!(mpm, RemoveNIPass()) # lower GC intrinsics @@ -325,7 +328,7 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op add!(fpm, GPULowerGCFramePass(job)) end if job.config.libraries - add!(mpm, GPULinkRuntimePass(job)) + add!(mpm, GPULinkRuntimePass(job, refs)) add!(mpm, GPULinkLibrariesPass(job)) add!(mpm, GPUFinishRuntimeIntrinsicsPass(job)) end @@ -475,6 +478,7 @@ GPULowerCPUFeaturesPass(job) = NewPMModulePass("GPULowerCPUFeatures", CPUFeature struct LinkRuntime job::CompilerJob + refs::HostReferences end function (self::LinkRuntime)(mod::LLVM.Module) self.job.config.libraries || return false @@ -483,15 +487,16 @@ function (self::LinkRuntime)(mod::LLVM.Module) # GC lowering can introduce new calls to GPU runtime functions after the runtime # was linked initially. Link again now so those calls resolve to definitions before # later intrinsic-lowering passes inspect or rewrite the runtime call graph. - runtime = load_runtime(self.job) + runtime, runtime_refs = load_runtime(self.job) # `RemoveNIPass` stripped non-integral address spaces from `mod`'s datalayout, but the # cached runtime kept them; align it (as with target libraries) to avoid a warning. triple!(runtime, triple(mod)) datalayout!(runtime, datalayout(mod)) - link!(mod, runtime; only_needed=true) + link_with_host_references!(mod, self.refs, runtime, runtime_refs; only_needed=true) return true end -GPULinkRuntimePass(job) = NewPMModulePass("GPULinkRuntime", LinkRuntime(job)) +GPULinkRuntimePass(job, refs=HostReferences()) = + NewPMModulePass("GPULinkRuntime", LinkRuntime(job, refs)) struct LinkLibraries job::CompilerJob diff --git a/src/rtlib.jl b/src/rtlib.jl index b60ba806..84c1aa3a 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -68,14 +68,15 @@ end # entirely. On 1.10 it is cached for the duration of the session. mutable struct RuntimeFunctionResults bitcode::Union{Nothing,Vector{UInt8}} - RuntimeFunctionResults() = new(nothing) + host_references::HostReferences + RuntimeFunctionResults() = new(nothing, HostReferences()) end # Compile a single runtime function and link it into `mod`. The renamed bitcode is # memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache # below additionally avoids repeating the parse-and-link work within a session. -function emit_function!(mod, config::CompilerConfig, source::MethodInstance, method, - world::UInt) +function emit_function!(mod, refs::HostReferences, config::CompilerConfig, + source::MethodInstance, method, world::UInt) name = method.llvm_name rt_job = CompilerJob(source, config, world) @@ -83,7 +84,9 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met # inference itself. ci, res = runtime_function_results(rt_job) if res !== nothing && res.bitcode !== nothing - link!(mod, parse(LLVM.Module, MemoryBuffer(res.bitcode))) + cached_refs = copy_host_references(res.host_references) + link_with_host_references!(mod, refs, + parse(LLVM.Module, MemoryBuffer(res.bitcode)), cached_refs) ci === nothing && (ci = runtime_code_instance(rt_job)) return ci::CodeInstance end @@ -97,13 +100,9 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) + prune_dead_host_reference_slots!(new_mod, meta.host_references) - # Resolve constgv mappings before their metadata is discarded. Dedicated Bool - # globals are resolved after linking into the toplevel module. - if !isempty(meta.gv_to_value) - portable = relocate_gvs!(new_mod, meta.gv_to_value) - portable || mark_session_dependent!(rt_job) - end + meta.host_references.embedded_pointer && mark_session_dependent!(rt_job) # rename to the final `gpu_*` name on the per-function module, so the cached bitcode # is immediately link-ready (no per-session rename pass on a cache hit). @@ -120,8 +119,9 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met ci === nothing && (ci = runtime_code_instance(rt_job)) res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) res.bitcode = take!(io) + res.host_references = copy_host_references(meta.host_references) - link!(mod, new_mod) + link_with_host_references!(mod, refs, new_mod, meta.host_references) return ci::CodeInstance end @@ -170,13 +170,14 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) mod = LLVM.Module("GPUCompiler run-time library") sources = MethodInstance[] code_instances = CodeInstance[] + refs = HostReferences() for method in values(Runtime.methods) resolved = runtime_method_instance(job, method) resolved === nothing && continue source = resolved push!(sources, source) - push!(code_instances, emit_function!(mod, config, source, method, job.world)) + push!(code_instances, emit_function!(mod, refs, config, source, method, job.world)) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -184,7 +185,7 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) # removes Julia address spaces, which would then lead to type mismatches when using # functions from the runtime library from IR that has not been stripped of AS info. - return mod, sources, code_instances + return mod, sources, code_instances, refs end # Session-local cache of assembled runtime libraries, keyed by @@ -203,6 +204,7 @@ mutable struct RuntimeLibrary sources::Vector{MethodInstance} code_instances::Vector{CodeInstance} validated_world::UInt + host_references::HostReferences end function runtime_library_valid(lib::RuntimeLibrary, @nospecialize(job::CompilerJob)) @@ -235,14 +237,16 @@ const runtime_libs_lock = ReentrantLock() cached = Base.@lock runtime_libs_lock begin cached = get(runtime_libs, key, nothing) if cached === nothing || !runtime_library_valid(cached, job) - lib, sources, code_instances = build_runtime(job, config) + lib, sources, code_instances, host_references = build_runtime(job, config) io = IOBuffer() write(io, lib) - cached = RuntimeLibrary(take!(io), sources, code_instances, job.world) + cached = RuntimeLibrary(take!(io), sources, code_instances, job.world, + host_references) runtime_libs[key] = cached end cached end - return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true) + return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true), + copy_host_references(cached.host_references) end diff --git a/src/validation.jl b/src/validation.jl index c8e97638..9d6d396e 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -162,8 +162,8 @@ end # `show` via `showerror`, avoiding the default field-dump that derefs disposed IR Base.show(io::IO, err::InvalidIRError) = showerror(io, err) -function check_ir(job, args...) - errors = check_ir!(job, IRError[], args...) +function check_ir(job, mod::LLVM.Module, refs::HostReferences=HostReferences()) + errors = check_ir!(job, IRError[], mod, refs) unique!(errors) if !isempty(errors) throw(InvalidIRError(job, errors)) @@ -172,9 +172,9 @@ function check_ir(job, args...) return end -function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module) +function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module, refs::HostReferences) for f in functions(mod) - check_ir!(job, errors, f) + check_ir!(job, errors, f, refs) end # custom validation @@ -183,10 +183,10 @@ function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module) return errors end -function check_ir!(job, errors::Vector{IRError}, f::LLVM.Function) +function check_ir!(job, errors::Vector{IRError}, f::LLVM.Function, refs::HostReferences) for bb in blocks(f), inst in instructions(bb) if isa(inst, LLVM.CallInst) - check_ir!(job, errors, inst) + check_ir!(job, errors, inst, refs) elseif isa(inst, LLVM.LoadInst) check_ir!(job, errors, inst) end @@ -197,6 +197,28 @@ end const libjulia = Ref{Ptr{Cvoid}}(C_NULL) +function referenced_object(value, refs::HostReferences) + while value isa ConstantExpr && + opcode(value) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) + value = first(operands(value)) + end + if value isa LLVM.LoadInst + source = first(operands(value)) + while source isa ConstantExpr && + opcode(source) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) + source = first(operands(source)) + end + if source isa GlobalVariable + ref = get(refs.slots, LLVM.name(source), nothing) + ref isa JuliaValueRef && return Some(ref.value) + end + elseif value isa ConstantExpr && opcode(value) == LLVM.API.LLVMIntToPtr + ptr = Ptr{Cvoid}(convert(Int, first(operands(value)))) + return Some(Base.unsafe_pointer_to_objref(ptr)) + end + return nothing +end + function check_ir!(job, errors::Vector{IRError}, inst::LLVM.LoadInst) bt = backtrace(inst) src = operands(inst)[1] @@ -221,7 +243,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.LoadInst) return errors end -function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) +function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::HostReferences) bt = backtrace(inst) dest = called_operand(inst) if isa(dest, LLVM.Function) @@ -233,11 +255,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) elseif fn == "jl_get_binding_or_error" || fn == "ijl_get_binding_or_error" try m, sym = arguments(inst) - sym = first(operands(sym::ConstantExpr))::ConstantInt - sym = convert(Int, sym) - sym = Ptr{Cvoid}(sym) - sym = Base.unsafe_pointer_to_objref(sym) - push!(errors, (DELAYED_BINDING, bt, sym)) + ref = referenced_object(sym, refs) + ref === nothing && error("Unknown binding") + push!(errors, (DELAYED_BINDING, bt, something(ref))) catch e @safe_debug "Decoding arguments to jl_get_binding_or_error failed" inst bb=LLVM.parent(inst) push!(errors, (DELAYED_BINDING, bt, nothing)) @@ -246,10 +266,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) fn == "jl_get_binding_value_seqcst" || fn == "ijl_get_binding_value_seqcst" try # pry the binding from the IR - expr = arguments(inst)[1]::ConstantExpr - expr = first(operands(expr))::ConstantInt # get rid of inttoptr - ptr = Ptr{Any}(convert(Int, expr)) - obj = Base.unsafe_pointer_to_objref(ptr) + ref = referenced_object(arguments(inst)[1], refs) + ref === nothing && error("Unknown binding") + obj = something(ref) push!(errors, (DELAYED_BINDING, bt, obj.globalref)) catch e @safe_debug "Decoding arguments to jl_reresolve_binding_value_seqcst failed" inst bb=LLVM.parent(inst) @@ -258,10 +277,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) elseif startswith(fn, "tojlinvoke") try fun, args, nargs = arguments(inst) - fun = first(operands(fun::ConstantExpr))::ConstantInt - fun = convert(Int, fun) - fun = Ptr{Cvoid}(fun) - fun = Base.unsafe_pointer_to_objref(fun)::Base.Function + ref = referenced_object(fun, refs) + ref === nothing && error("Unknown function") + fun = something(ref)::Base.Function push!(errors, (DYNAMIC_CALL, bt, fun)) # XXX: an invoke trampoline happens when codegen doesn't have access to code # which suggests a GPUCompiler.jl bug. throw an error instead? @@ -279,10 +297,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) end try fun, args, nargs, meth = arguments(inst) - meth = first(operands(meth::ConstantExpr))::ConstantInt - meth = convert(Int, meth) - meth = Ptr{Cvoid}(meth) - meth = Base.unsafe_pointer_to_objref(meth)::Core.MethodInstance + ref = referenced_object(meth, refs) + ref === nothing && error("Unknown method instance") + meth = something(ref)::Core.MethodInstance push!(errors, (DYNAMIC_CALL, bt, meth.def)) catch e @safe_debug "Decoding arguments to jl_invoke failed" inst bb=LLVM.parent(inst) @@ -291,10 +308,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) elseif fn == "jl_apply_generic" || fn == "ijl_apply_generic" try f, args, nargs = arguments(inst) - f = first(operands(f))::ConstantInt # get rid of inttoptr - f = convert(Int, f) - f = Ptr{Cvoid}(f) - f = Base.unsafe_pointer_to_objref(f) + ref = referenced_object(f, refs) + ref === nothing && error("Unknown function") + f = something(ref) push!(errors, (DYNAMIC_CALL, bt, f)) catch e @safe_debug "Decoding arguments to jl_apply_generic failed" inst bb=LLVM.parent(inst) diff --git a/test/native.jl b/test/native.jl index 834c0f20..0ac51ea4 100644 --- a/test/native.jl +++ b/test/native.jl @@ -73,17 +73,9 @@ end @test only(other_mis).def in methods(mod.inner) if VERSION >= v"1.12" - @test length(meta.gv_to_value) == 1 - for (k, v) in meta.gv_to_value - @test v != C_NULL - end + @test length(meta.host_references.slots) == 1 + @test only(values(meta.host_references.slots)) isa GPUCompiler.JuliaValueRef end - # TODO: Global values get privatized, so we can't find them by name anymore. - # %.not = icmp eq ptr %"sym::Symbol", inttoptr (i64 140096668482288 to ptr), !dbg !38 - # for (name, v) in meta.gv_to_value - # gv = globals(ir)[name] - # @test LLVM.initializer(gv) === v - # end end end @@ -273,11 +265,10 @@ end end end - @testset "runtime constgv relocation" begin + @testset "runtime host references" begin # runtime functions like `box_bool` may reference Julia singletons through - # `julia.constgv` globals. Their session-absolute addresses must be baked into - # the cached runtime bitcode when it is built: only kernel modules go through - # `relocate_gvs!`, so a slot left null here would stay null on the device. + # `julia.constgv` globals. Keep their Julia identities with the cached bitcode + # so the final kernel can resolve them in its own session. job, _ = Native.create_job(identity, (Nothing,)) JuliaContext() do ctx GPUCompiler.load_runtime(job) @@ -292,7 +283,8 @@ end isempty(uses(gv)) && continue used += 1 init = LLVM.initializer(gv) - @test init !== nothing && !LLVM.isnull(init) + @test init === nothing + @test haskey(lib.host_references.slots, LLVM.name(gv)) end @static if VERSION >= v"1.12-" # on older versions, Julia bakes addresses without tagging globals @@ -737,6 +729,12 @@ end # actually loaded from, leaving called functions alone. job, _ = Native.create_job(identity, (Nothing,)) JuliaContext() do ctx + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + word_ptr = ptr("i8") + word_ptr_ptr = ptr(word_ptr) + function_word_ptr(name) = GPUCompiler.supports_typed_pointers(ctx) ? + "i64* bitcast (i64 ()* @$name to i64*)" : "ptr @$name" + mod = parse(LLVM.Module, """ declare void @jl_get_pgcstack_resolved() @@ -746,6 +744,118 @@ end }""") GPUCompiler.prepare_execution!(job, mod) @test haskey(functions(mod), "jl_get_pgcstack_resolved") + + mod = parse(LLVM.Module, """ + @jl_float32_type = external global $word_ptr + + define $word_ptr @entry() { + %value = load $word_ptr, $word_ptr_ptr @jl_float32_type + ret $word_ptr %value + }""") + GPUCompiler.prepare_execution!(job, mod) + @test !occursin("load $word_ptr, $word_ptr_ptr @jl_float32_type", string(mod)) + + mod = parse(LLVM.Module, """ + @jl_float32_type = external global $word_ptr + + define $word_ptr @entry() { + %value = load $word_ptr, $word_ptr_ptr @jl_float32_type + ret $word_ptr %value + }""") + refs = GPUCompiler.HostReferences() + @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) + name = only(keys(refs.slots)) + @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test occursin("externally_initialized global i64 0", string(mod)) + + mod = parse(LLVM.Module, """ + declare i64 @jl_float32_type() + + define i64 @entry() { + %value = load i64, $(function_word_ptr("jl_float32_type")) + ret i64 %value + }""") + refs = GPUCompiler.HostReferences() + @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) + name = only(keys(refs.slots)) + @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test occursin("externally_initialized global i64 0", string(mod)) + end +end + +@testset "host reference linking" begin + JuliaContext() do ctx + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + + function slot_module(name, entry) + parse(LLVM.Module, """ + @$name = external global i64 + + define i64 @$entry() { + %value = load i64, $(ptr("i64")) @$name + ret i64 %value + }""") + end + + refs(name, value) = + GPUCompiler.HostReferences(Dict(name => GPUCompiler.JuliaValueRef(value)), false) + + # Three unrelated values requested the same slot name. Each load keeps its own + # relocation after linking, including the second fresh-name suffix. + dest = slot_module("slot", "first") + dest_refs = refs("slot", :first) + for (entry, value) in (("second", :second), ("third", :third)) + src = slot_module("slot", entry) + src_refs = refs("slot", value) + GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) + end + @test Set(keys(dest_refs.slots)) == + Set(("slot", "slot_gpucompiler", "slot_gpucompiler_1")) + @test Set(ref.value for ref in values(dest_refs.slots)) == Set((:first, :second, :third)) + + # A global outside the metadata table is still part of LLVM's namespace. + dest = slot_module("slot", "occupied") + dest_refs = GPUCompiler.HostReferences() + src = slot_module("slot", "source") + src_refs = refs("slot", :source) + GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) + @test only(keys(dest_refs.slots)) == "slot_gpucompiler" + + # Equal Julia identities deliberately share a single slot. + dest = slot_module("slot", "first") + dest_refs = refs("slot", :shared) + src = slot_module("slot", "second") + src_refs = refs("slot", :shared) + GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) + @test collect(keys(dest_refs.slots)) == ["slot"] + @test occursin("@slot = external global i64", string(dest)) + + # `only_needed` must keep metadata for imported slots and discard metadata for + # source globals that the LLVM linker did not import. + dest = parse(LLVM.Module, """ + declare i64 @source() + + define i64 @entry() { + %value = call i64 @source() + ret i64 %value + }""") + src = parse(LLVM.Module, """ + @used = external global i64 + @unused = external global i64 + + define i64 @source() { + %value = load i64, $(ptr("i64")) @used + ret i64 %value + }""") + src_refs = GPUCompiler.HostReferences(Dict( + "used" => GPUCompiler.JuliaValueRef(:used), + "unused" => GPUCompiler.JuliaValueRef(:unused), + ), false) + dest_refs = GPUCompiler.HostReferences() + GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs; + only_needed=true) + @test collect(keys(dest_refs.slots)) == ["used"] + @test only(values(dest_refs.slots)).value === :used end end diff --git a/test/ptx.jl b/test/ptx.jl index e98f55d8..5a5307e3 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -40,8 +40,25 @@ end ir = sprint() do io PTX.code_llvm(io, mod.kernel, Tuple{Symbol}; dump_module=true) end - addr = UInt64(pointer_from_objref(:var)) - @test occursin(string(addr), ir) + # Julia 1.10 may bake the Symbol pointer into the generated IR before exposing it to + # GPUCompiler. Newer versions provide a symbolic global that we can preserve and relocate. + @static if VERSION >= v"1.11" + @test occursin("@jl_sym_var_", ir) + else + @test occursin("@jl_sym_var_", ir) || occursin("inttoptr", ir) + end + @test !occursin("@\"jl_sym#", ir) +end + +@testset "Julia value global names" begin + # Julia's external codegen can name the declaration for this Symbol with `#`. + # The final PTX path must see the sanitized Julia value global, not the original + # declaration name rejected by NVPTX. + mod = @eval module $(gensym()) + const unusual_symbol = Symbol("value#global") + kernel(name::Symbol) = (name === unusual_symbol; return) + end + @test PTX.code_execution(mod.kernel, Tuple{Symbol}) !== nothing end @testset "boxed Bool singleton relocation" begin From 1c83258bbb34f6f40cdfe883e43feca6cb03ebc3 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:25:36 +0200 Subject: [PATCH 02/45] Mark resolved host-reference slots constant --- src/jlgen.jl | 1 + test/native.jl | 1 + 2 files changed, 2 insertions(+) diff --git a/src/jlgen.jl b/src/jlgen.jl index 8c295630..cb647293 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -828,6 +828,7 @@ function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) val = host_reference_slot_initializer(gv, value) initializer!(gv, val) linkage!(gv, LLVM.API.LLVMPrivateLinkage) + constant!(gv, true) refs.embedded_pointer = true end empty!(refs.slots) diff --git a/test/native.jl b/test/native.jl index 0ac51ea4..9e78562e 100644 --- a/test/native.jl +++ b/test/native.jl @@ -754,6 +754,7 @@ end }""") GPUCompiler.prepare_execution!(job, mod) @test !occursin("load $word_ptr, $word_ptr_ptr @jl_float32_type", string(mod)) + @test occursin(r"private .*constant", string(mod)) mod = parse(LLVM.Module, """ @jl_float32_type = external global $word_ptr From 6d21d0f6959a295c3b3464514b827e31d9a99b70 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:26:28 +0200 Subject: [PATCH 03/45] Harden JuliaValueRef resolution against re-boxing --- src/interface.jl | 21 ++++++++++++++++----- test/native.jl | 12 ++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 4253c6b4..dee56a95 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -432,12 +432,19 @@ function cached_results end """ JuliaValueRef(value) -A Julia value used as the serializable identity of a host reference. Resolve it in the active -session with [`resolve_host_reference`](@ref); a loader that writes the resulting address into -device storage must keep `value` rooted for as long as that storage remains reachable. +A Julia value with a stable address (a heap object, symbol, or singleton), used as the +serializable identity of a host reference. Resolve it in the active session with +[`resolve_host_reference`](@ref); a loader that writes the resulting address into device +storage must keep `value` rooted for as long as that storage remains reachable. """ struct JuliaValueRef value::Any + + function JuliaValueRef(value) + isbitstype(typeof(value)) && sizeof(value) > 0 && + error("JuliaValueRef requires an object with a stable address") + new(value) + end end """ @@ -485,8 +492,12 @@ same_host_reference(::HostReference, ::HostReference) = false Resolve `ref` to its current-session word. Loader-based backends use this after loading the object and before exposing a callable function. """ -resolve_host_reference(ref::JuliaValueRef) = - UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), ref.value)) +function resolve_host_reference(ref::JuliaValueRef) + box = Any[ref.value] + GC.@preserve box begin + return unsafe_load(Base.unsafe_convert(Ptr{UInt}, pointer(box))) + end +end function resolve_host_reference(ref::CGlobalRef) address = ccall(:jl_cglobal, Any, (Any, Any), String(ref.symbol), UInt) return unsafe_load(address) diff --git a/test/native.jl b/test/native.jl index 9e78562e..fcc3c259 100644 --- a/test/native.jl +++ b/test/native.jl @@ -723,6 +723,18 @@ end end end +@testset "host reference resolution" begin + sym = :host_reference_probe + @test GPUCompiler.resolve_host_reference(GPUCompiler.JuliaValueRef(sym)) == + UInt(pointer_from_objref(sym)) + + singleton = nothing + @test GPUCompiler.resolve_host_reference(GPUCompiler.JuliaValueRef(singleton)) == + UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), singleton)) + + @test_throws ErrorException GPUCompiler.JuliaValueRef(1.5) +end + @testset "CPU reference resolution" begin # JIT-private symbols like `jl_get_pgcstack_resolved` (JuliaLang/julia#61527) cannot # be looked up using `jl_cglobal`, so we should only resolve bindings that are From 1a984d97477c7bd32e54760afe7079ab1a7a7d52 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:26:41 +0200 Subject: [PATCH 04/45] Clean up host-reference comments --- src/jlgen.jl | 2 ++ src/mcgen.jl | 3 +-- src/validation.jl | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/jlgen.jl b/src/jlgen.jl index cb647293..4745af94 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -768,6 +768,8 @@ function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) continue end + # jl_get_llvm_gvs and jl_get_llvm_gv_inits report an initializer for every + # mapped global, so a null here means those maps are out of sync. init == C_NULL && error("Missing Julia object for global '$name'") obj = Base.unsafe_pointer_to_objref(init) if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) diff --git a/src/mcgen.jl b/src/mcgen.jl index b7a2438e..469a9c77 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -64,7 +64,7 @@ function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) function replace_bindings!(value) changed = false - for use in uses(value) + for use in collect(uses(value)) val = user(use) if isa(val, LLVM.ConstantExpr) # recurse @@ -85,7 +85,6 @@ function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) replace_uses!(val, replacement) end erase!(val) - # FIXME: iterator invalidation? changed = true end end diff --git a/src/validation.jl b/src/validation.jl index 9d6d396e..9cd0514c 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -198,6 +198,7 @@ end const libjulia = Ref{Ptr{Cvoid}}(C_NULL) function referenced_object(value, refs::HostReferences) + # This is best-effort: optimized shapes fall back to the unknown-binding error path. while value isa ConstantExpr && opcode(value) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) value = first(operands(value)) From 40e0fe60d6900d53b0d4869b42d6ab2078b80b33 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:27:44 +0200 Subject: [PATCH 05/45] Emit runtime-global slots as declarations until lowering --- src/mcgen.jl | 3 --- test/native.jl | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/mcgen.jl b/src/mcgen.jl index 469a9c77..7593f590 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -51,9 +51,6 @@ function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) if slot === nothing name = safe_name("gpu_" * fn) slot = GlobalVariable(mod, host_reference_word_type(), name) - initializer!(slot, null(global_value_type(slot))) - linkage!(slot, LLVM.API.LLVMExternalLinkage) - extinit!(slot, true) actual_name = LLVM.name(slot) haskey(self.refs.slots, actual_name) && error("Duplicate Julia runtime global slot '$actual_name'") diff --git a/test/native.jl b/test/native.jl index fcc3c259..8ada2ea7 100644 --- a/test/native.jl +++ b/test/native.jl @@ -779,6 +779,8 @@ end @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) name = only(keys(refs.slots)) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test occursin("@$name = external global i64", string(mod)) + GPUCompiler.emit_host_reference_slots!(mod, refs) @test occursin("externally_initialized global i64 0", string(mod)) mod = parse(LLVM.Module, """ @@ -792,7 +794,7 @@ end @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) name = only(keys(refs.slots)) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) - @test occursin("externally_initialized global i64 0", string(mod)) + @test occursin("@$name = external global i64", string(mod)) end end From c51756aa229210b0dc34bec5ffce94df4530f457 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:30:59 +0200 Subject: [PATCH 06/45] Add declaration-based host-reference lowering for JIT loaders --- src/interface.jl | 14 ++++++++++---- src/jlgen.jl | 46 ++++++++++++++++++++++++++++++++++++++++++++++ test/native.jl | 19 +++++++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index dee56a95..e6b644ab 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -507,9 +507,14 @@ end lower_host_references!(job, mod, refs) Lower compiler-managed Julia value globals and Julia runtime globals for `job`'s final -backend representation. The default lowering resolves every live reference in the current -Julia session. Backends with a loader relocation mechanism may call -[`emit_host_reference_slots!`](@ref) instead, then resolve and patch those slots after loading. +backend representation. The default [`resolve_host_reference_slots!`](@ref) lowering resolves +every live reference in the current Julia session, making the result session-dependent. +Backends with a per-module loader namespace, such as a `CuModule`, may call +[`emit_host_reference_slots!`](@ref) and patch the resulting definitions after loading. +Shared-namespace JIT loaders may call [`emit_host_reference_declarations!`](@ref), define the +remaining declarations before loading, and resolve libjulia globals by name. Because slot names +are unique only within one compilation, such a loader must use a fresh namespace per object or +uniquify names at link time. """ function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, refs::HostReferences) @@ -638,7 +643,8 @@ end # HAS_INTEGRATED_CACHE @public GPUCompilerCacheToken, cache_owner, cached_results @public JuliaValueRef, CGlobalRef, HostReference, HostReferences -@public lower_host_references!, emit_host_reference_slots!, resolve_host_reference +@public lower_host_references!, emit_host_reference_slots!, emit_host_reference_declarations! +@public resolve_host_reference # the method table to use # diff --git a/src/jlgen.jl b/src/jlgen.jl index 4745af94..6c0a016f 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -860,6 +860,52 @@ function emit_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) return end +""" + emit_host_reference_declarations!(mod, refs) + +Lower host references for loaders that resolve symbols by name at link time. Julia runtime +globals are folded back into direct references to their libjulia symbols. Julia value globals +remain external, word-sized declarations; before loading the object, the loader must define +each remaining symbol to point at a cell containing [`resolve_host_reference`](@ref), and keep +the referenced values rooted while the code remains executable. +""" +function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) + check_host_reference_slots!(mod, refs) + mod_gvs = globals(mod) + for (name, ref) in collect(refs.slots) + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + + if ref isa CGlobalRef + symbol = String(ref.symbol) + if symbol == name + initializer!(gv, nothing) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + else + replacement = if haskey(globals(mod), symbol) + globals(mod)[symbol] + elseif haskey(functions(mod), symbol) + functions(mod)[symbol] + else + GlobalVariable(mod, host_reference_word_type(), symbol) + end + replacement = value_type(replacement) == value_type(gv) ? replacement : + const_pointercast(replacement, value_type(gv)) + replace_uses!(gv, replacement) + erase!(gv) + end + delete!(refs.slots, name) + else + isdeclaration(gv) || + error("Julia value host reference slot '$name' must be a declaration") + linkage!(gv, LLVM.API.LLVMExternalLinkage) + end + end + return +end + function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, src_mod::LLVM.Module, src_refs::HostReferences; only_needed=false) diff --git a/test/native.jl b/test/native.jl index 8ada2ea7..599c106c 100644 --- a/test/native.jl +++ b/test/native.jl @@ -795,6 +795,25 @@ end name = only(keys(refs.slots)) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) + GPUCompiler.emit_host_reference_declarations!(mod, refs) + @test isempty(refs.slots) + @test !haskey(globals(mod), name) + @test occursin("load i64, $(function_word_ptr("jl_float32_type"))", string(mod)) + + mod = parse(LLVM.Module, """ + @jl_float32_type = external global $word_ptr + + define $word_ptr @entry() { + %value = load $word_ptr, $word_ptr_ptr @jl_float32_type + ret $word_ptr %value + }""") + refs = GPUCompiler.HostReferences() + @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) + name = only(keys(refs.slots)) + GPUCompiler.emit_host_reference_declarations!(mod, refs) + @test isempty(refs.slots) + @test !haskey(globals(mod), name) + @test occursin(r"load i64, .*@jl_float32_type", string(mod)) end end From 42be656e963ce44935e00d01e58808685e4211a3 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:33:12 +0200 Subject: [PATCH 07/45] Test postponed relocation with an ORC loader in the native back-end --- test/helpers/native.jl | 52 ++++++++++++++++++++++++++++++++++++++---- test/native.jl | 31 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 4 deletions(-) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 63d8a0f3..d0f28fe7 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -1,6 +1,7 @@ module Native using ..GPUCompiler +using LLVM import ..TestRuntime # local method table for device functions @@ -9,9 +10,11 @@ Base.Experimental.@MethodTable(test_method_table) struct CompilerParams <: AbstractCompilerParams entry_safepoint::Bool method_table + relocatable::Bool - CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table) = - new(entry_safepoint, method_table) + CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table, + relocatable::Bool=false) = + new(entry_safepoint, method_table, relocatable) end module Runtime end @@ -21,17 +24,58 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint +function GPUCompiler.lower_host_references!(@nospecialize(job::NativeCompilerJob), + mod::LLVM.Module, + refs::GPUCompiler.HostReferences) + if job.config.params.relocatable + GPUCompiler.emit_host_reference_declarations!(mod, refs) + else + invoke(GPUCompiler.lower_host_references!, + Tuple{CompilerJob,LLVM.Module,GPUCompiler.HostReferences}, job, mod, refs) + end +end function create_job(@nospecialize(func), @nospecialize(types); - entry_safepoint::Bool=false, method_table=test_method_table, kwargs...) + entry_safepoint::Bool=false, method_table=test_method_table, + relocatable::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = NativeCompilerTarget(;jlruntime=true) - params = CompilerParams(entry_safepoint, method_table) + params = CompilerParams(entry_safepoint, method_table, relocatable) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end +function load(obj::Vector{UInt8}, entry::String, refs::GPUCompiler.HostReferences) + lljit = LLJIT() + try + jd = JITDylib(lljit) + prefix = LLVM.get_prefix(lljit) + add!(jd, LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)) + + cells = Vector{UInt}(undef, length(refs.slots)) + roots = Any[] + pairs = LLVM.API.LLVMOrcCSymbolMapPair[] + for (i, (name, ref)) in enumerate(refs.slots) + cells[i] = GPUCompiler.resolve_host_reference(ref) + ref isa GPUCompiler.JuliaValueRef && push!(roots, ref.value) + symbol = LLVM.API.LLVMJITEvaluatedSymbol( + reinterpret(UInt, pointer(cells, i)), + LLVM.API.LLVMJITSymbolFlags( + LLVM.API.LLVMJITSymbolGenericFlagsExported, 0)) + push!(pairs, LLVM.API.LLVMOrcCSymbolMapPair(mangle(lljit, name), symbol)) + end + isempty(pairs) || LLVM.define(jd, LLVM.absolute_symbols(pairs)) + + add!(lljit, jd, MemoryBuffer(obj)) + addr = lookup(lljit, entry) + return pointer(addr), (lljit, cells, roots) + catch + dispose(lljit) + rethrow() + end +end + function code_typed(@nospecialize(func), @nospecialize(types); kwargs...) job, kwargs = create_job(func, types; kwargs...) GPUCompiler.code_typed(job; kwargs...) diff --git a/test/native.jl b/test/native.jl index 599c106c..ef66ebaf 100644 --- a/test/native.jl +++ b/test/native.jl @@ -735,6 +735,37 @@ end @test_throws ErrorException GPUCompiler.JuliaValueRef(1.5) end +@testset "postponed relocation" begin + if VERSION >= v"1.12" + mod = @eval module $(gensym()) + f(x::Symbol) = x === :host_ref_probe + end + job, _ = Native.create_job(mod.f, (Symbol,); relocatable=true) + JuliaContext() do ctx + obj, meta = GPUCompiler.compile(:obj, job) + refs = meta.host_references + @test !isempty(refs.slots) + @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(refs.slots)) + @test !refs.embedded_pointer + @test all(name -> isdeclaration(globals(meta.ir)[name]), keys(refs.slots)) + + bytes = Vector{UInt8}(codeunits(obj)) + entry = LLVM.name(meta.entry) + fptr, keepalive = Native.load(bytes, entry, refs) + try + GC.@preserve keepalive begin + @test ccall(fptr, Bool, (Any,), :host_ref_probe) + @test !ccall(fptr, Bool, (Any,), :other) + end + finally + dispose(first(keepalive)) + end + + @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.HostReferences()) + end + end +end + @testset "CPU reference resolution" begin # JIT-private symbols like `jl_get_pgcstack_resolved` (JuliaLang/julia#61527) cannot # be looked up using `jl_cglobal`, so we should only resolve bindings that are From c629367597aa9c8012816a1476a512f9a979e065 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 14:57:47 +0200 Subject: [PATCH 08/45] Clarify host-reference lowering names --- src/interface.jl | 24 ++++++++++-------------- src/jlgen.jl | 22 +++++++++++++--------- test/native.jl | 2 +- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index e6b644ab..345a5488 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -450,8 +450,8 @@ end """ CGlobalRef(symbol) -A libjulia C data global. Resolving this reference calls `jl_cglobal` for `symbol` and loads -the word stored at that address. +A named libjulia C data global. Resolution returns the word stored in that global in the +current Julia process. """ struct CGlobalRef symbol::Symbol @@ -489,8 +489,7 @@ same_host_reference(::HostReference, ::HostReference) = false """ resolve_host_reference(ref) -> UInt -Resolve `ref` to its current-session word. Loader-based backends use this after loading the -object and before exposing a callable function. +Resolve a host reference to its word in the current Julia process. """ function resolve_host_reference(ref::JuliaValueRef) box = Any[ref.value] @@ -506,15 +505,11 @@ end """ lower_host_references!(job, mod, refs) -Lower compiler-managed Julia value globals and Julia runtime globals for `job`'s final -backend representation. The default [`resolve_host_reference_slots!`](@ref) lowering resolves -every live reference in the current Julia session, making the result session-dependent. -Backends with a per-module loader namespace, such as a `CuModule`, may call -[`emit_host_reference_slots!`](@ref) and patch the resulting definitions after loading. -Shared-namespace JIT loaders may call [`emit_host_reference_declarations!`](@ref), define the -remaining declarations before loading, and resolve libjulia globals by name. Because slot names -are unique only within one compilation, such a loader must use a fresh namespace per object or -uniquify names at link time. +Backend hook for lowering live host references before object emission. + +The default implementation resolves them in the current Julia process, making the result +session-dependent. Loaders may instead emit module-owned definitions that are patched after +loading, or external declarations that are defined before loading. """ function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, refs::HostReferences) @@ -643,7 +638,8 @@ end # HAS_INTEGRATED_CACHE @public GPUCompilerCacheToken, cache_owner, cached_results @public JuliaValueRef, CGlobalRef, HostReference, HostReferences -@public lower_host_references!, emit_host_reference_slots!, emit_host_reference_declarations! +@public lower_host_references!, emit_host_reference_definitions! +@public emit_host_reference_declarations! @public resolve_host_reference # the method table to use diff --git a/src/jlgen.jl b/src/jlgen.jl index 6c0a016f..3d036971 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -838,12 +838,12 @@ function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) end """ - emit_host_reference_slots!(mod, refs) + emit_host_reference_definitions!(mod, refs) -Prepare the globals named by `refs.slots` for loader-based lowering. Each remains a writable, -word-sized external symbol that the loader must resolve and patch after loading the object. +Emit host-reference slots as writable, null-initialized definitions. The loader must patch each +definition after loading the object. This requires a per-object symbol namespace. """ -function emit_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) +function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences) check_host_reference_slots!(mod, refs) mod_gvs = globals(mod) slots = GlobalVariable[] @@ -863,11 +863,15 @@ end """ emit_host_reference_declarations!(mod, refs) -Lower host references for loaders that resolve symbols by name at link time. Julia runtime -globals are folded back into direct references to their libjulia symbols. Julia value globals -remain external, word-sized declarations; before loading the object, the loader must define -each remaining symbol to point at a cell containing [`resolve_host_reference`](@ref), and keep -the referenced values rooted while the code remains executable. +Prepare host references for a loader that defines symbols before loading an object. + +Runtime-global slots are restored to direct references to their named libjulia globals. Julia +value slots remain external word-sized declarations; the loader must define each symbol to +point at a cell containing [`resolve_host_reference`](@ref) and keep the cell and referenced +value alive while the code is executable. + +Slot names are unique only within one compilation. A shared JIT namespace must uniquify them +or use a separate namespace for each object. """ function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) check_host_reference_slots!(mod, refs) diff --git a/test/native.jl b/test/native.jl index ef66ebaf..7e324f5e 100644 --- a/test/native.jl +++ b/test/native.jl @@ -811,7 +811,7 @@ end name = only(keys(refs.slots)) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) - GPUCompiler.emit_host_reference_slots!(mod, refs) + GPUCompiler.emit_host_reference_definitions!(mod, refs) @test occursin("externally_initialized global i64 0", string(mod)) mod = parse(LLVM.Module, """ From 0c918e6c88945bee8d5624defaf0632bcc230163 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 15:26:24 +0200 Subject: [PATCH 09/45] Use JIT target machines for ORC relocation tests --- test/helpers/native.jl | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index d0f28fe7..a2d7bc38 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -35,6 +35,20 @@ function GPUCompiler.lower_host_references!(@nospecialize(job::NativeCompilerJob end end +function GPUCompiler.mcgen(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, + format=LLVM.API.LLVMAssemblyFile) + if job.config.params.relocatable + target = job.config.target + @dispose tm=JITTargetMachine(GPUCompiler.llvm_triple(target), target.cpu, + target.features) begin + return String(emit(tm, mod, format)) + end + else + return invoke(GPUCompiler.mcgen, Tuple{CompilerJob,LLVM.Module,Any}, + job, mod, format) + end +end + function create_job(@nospecialize(func), @nospecialize(types); entry_safepoint::Bool=false, method_table=test_method_table, relocatable::Bool=false, kwargs...) @@ -47,7 +61,7 @@ function create_job(@nospecialize(func), @nospecialize(types); end function load(obj::Vector{UInt8}, entry::String, refs::GPUCompiler.HostReferences) - lljit = LLJIT() + lljit = LLJIT(; tm=JITTargetMachine()) try jd = JITDylib(lljit) prefix = LLVM.get_prefix(lljit) From fc413a3648dcf54deea5e999ec3aa224ef1ba7b1 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 15:26:24 +0200 Subject: [PATCH 10/45] Make postponed relocation probe ABI-independent --- test/native.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/native.jl b/test/native.jl index 7e324f5e..887f3dbb 100644 --- a/test/native.jl +++ b/test/native.jl @@ -738,9 +738,9 @@ end @testset "postponed relocation" begin if VERSION >= v"1.12" mod = @eval module $(gensym()) - f(x::Symbol) = x === :host_ref_probe + f() = UInt(pointer_from_objref(:host_ref_probe)) end - job, _ = Native.create_job(mod.f, (Symbol,); relocatable=true) + job, _ = Native.create_job(mod.f, Tuple{}; relocatable=true) JuliaContext() do ctx obj, meta = GPUCompiler.compile(:obj, job) refs = meta.host_references @@ -751,11 +751,12 @@ end bytes = Vector{UInt8}(codeunits(obj)) entry = LLVM.name(meta.entry) + expected = GPUCompiler.resolve_host_reference(only(values(refs.slots))) fptr, keepalive = Native.load(bytes, entry, refs) try GC.@preserve keepalive begin - @test ccall(fptr, Bool, (Any,), :host_ref_probe) - @test !ccall(fptr, Bool, (Any,), :other) + actual = ccall(fptr, UInt, ()) + @test actual == expected end finally dispose(first(keepalive)) From 8093be9f69ff20b0f8bc974e72eae610c101ec80 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 15:37:46 +0200 Subject: [PATCH 11/45] Name the native test back-end JIT mode explicitly --- test/helpers/native.jl | 14 +++++++------- test/native.jl | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index a2d7bc38..8013f04c 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -10,11 +10,11 @@ Base.Experimental.@MethodTable(test_method_table) struct CompilerParams <: AbstractCompilerParams entry_safepoint::Bool method_table - relocatable::Bool + jit::Bool CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table, - relocatable::Bool=false) = - new(entry_safepoint, method_table, relocatable) + jit::Bool=false) = + new(entry_safepoint, method_table, jit) end module Runtime end @@ -27,7 +27,7 @@ GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.pa function GPUCompiler.lower_host_references!(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, refs::GPUCompiler.HostReferences) - if job.config.params.relocatable + if job.config.params.jit GPUCompiler.emit_host_reference_declarations!(mod, refs) else invoke(GPUCompiler.lower_host_references!, @@ -37,7 +37,7 @@ end function GPUCompiler.mcgen(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, format=LLVM.API.LLVMAssemblyFile) - if job.config.params.relocatable + if job.config.params.jit target = job.config.target @dispose tm=JITTargetMachine(GPUCompiler.llvm_triple(target), target.cpu, target.features) begin @@ -51,11 +51,11 @@ end function create_job(@nospecialize(func), @nospecialize(types); entry_safepoint::Bool=false, method_table=test_method_table, - relocatable::Bool=false, kwargs...) + jit::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = NativeCompilerTarget(;jlruntime=true) - params = CompilerParams(entry_safepoint, method_table, relocatable) + params = CompilerParams(entry_safepoint, method_table, jit) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end diff --git a/test/native.jl b/test/native.jl index 887f3dbb..0b8eac02 100644 --- a/test/native.jl +++ b/test/native.jl @@ -740,7 +740,7 @@ end mod = @eval module $(gensym()) f() = UInt(pointer_from_objref(:host_ref_probe)) end - job, _ = Native.create_job(mod.f, Tuple{}; relocatable=true) + job, _ = Native.create_job(mod.f, Tuple{}; jit=true) JuliaContext() do ctx obj, meta = GPUCompiler.compile(:obj, job) refs = meta.host_references From ec7978f94b302635fd5ac5fdfe17d751dc0bc0bd Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 18:56:08 +0200 Subject: [PATCH 12/45] Consolidate host-reference relocation lifecycle --- src/GPUCompiler.jl | 1 + src/interface.jl | 88 +------- src/jlgen.jl | 294 --------------------------- src/mcgen.jl | 77 ------- src/relocation.jl | 494 +++++++++++++++++++++++++++++++++++++++++++++ src/validation.jl | 23 --- 6 files changed, 496 insertions(+), 481 deletions(-) create mode 100644 src/relocation.jl diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index d1ef48e2..c25aa021 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -54,6 +54,7 @@ include("mangling.jl") # compiler interface and implementations include("interface.jl") +include("relocation.jl") include("error.jl") include("native.jl") include("ptx.jl") diff --git a/src/interface.jl b/src/interface.jl index 345a5488..83c91c78 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -429,93 +429,7 @@ the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock """ function cached_results end -""" - JuliaValueRef(value) - -A Julia value with a stable address (a heap object, symbol, or singleton), used as the -serializable identity of a host reference. Resolve it in the active session with -[`resolve_host_reference`](@ref); a loader that writes the resulting address into device -storage must keep `value` rooted for as long as that storage remains reachable. -""" -struct JuliaValueRef - value::Any - - function JuliaValueRef(value) - isbitstype(typeof(value)) && sizeof(value) > 0 && - error("JuliaValueRef requires an object with a stable address") - new(value) - end -end - -""" - CGlobalRef(symbol) - -A named libjulia C data global. Resolution returns the word stored in that global in the -current Julia process. -""" -struct CGlobalRef - symbol::Symbol -end - -""" - HostReference - -A serializable source for a host-derived word: either a [`JuliaValueRef`](@ref) or a -[`CGlobalRef`](@ref). -""" -const HostReference = Union{JuliaValueRef,CGlobalRef} - -""" - HostReferences(slots, embedded_pointer) - -Host-reference metadata accompanying a module. `slots` maps each object-code symbol to the -source of the word that must be written there. `embedded_pointer` conservatively records that -an unavoidable session pointer has been embedded, so the artifact cannot be serialized into a -package image even if `slots` is empty. -""" -mutable struct HostReferences - slots::Dict{String,HostReference} - embedded_pointer::Bool -end - -HostReferences() = HostReferences(Dict{String,HostReference}(), false) -copy_host_references(refs::HostReferences) = - HostReferences(copy(refs.slots), refs.embedded_pointer) - -same_host_reference(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value -same_host_reference(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol -same_host_reference(::HostReference, ::HostReference) = false - -""" - resolve_host_reference(ref) -> UInt - -Resolve a host reference to its word in the current Julia process. -""" -function resolve_host_reference(ref::JuliaValueRef) - box = Any[ref.value] - GC.@preserve box begin - return unsafe_load(Base.unsafe_convert(Ptr{UInt}, pointer(box))) - end -end -function resolve_host_reference(ref::CGlobalRef) - address = ccall(:jl_cglobal, Any, (Any, Any), String(ref.symbol), UInt) - return unsafe_load(address) -end - -""" - lower_host_references!(job, mod, refs) - -Backend hook for lowering live host references before object emission. - -The default implementation resolves them in the current Julia process, making the result -session-dependent. Loaders may instead emit module-owned definitions that are patched after -loading, or external declarations that are defined before loading. -""" -function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences) - resolve_host_reference_slots!(mod, refs) -end - +# Host-reference types and the backend lowering hook live in relocation.jl. @static if HAS_INTEGRATED_CACHE """ diff --git a/src/jlgen.jl b/src/jlgen.jl index 3d036971..9ecbffbf 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -756,300 +756,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end -function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - refs = HostReferences() - mod_gvs = globals(mod) - for (name, init) in gv_to_value - haskey(mod_gvs, name) || continue - gv = mod_gvs[name] - cur = initializer(gv) - if !(cur === nothing || LLVM.isnull(cur)) - refs.embedded_pointer = true - continue - end - - # jl_get_llvm_gvs and jl_get_llvm_gv_inits report an initializer for every - # mapped global, so a null here means those maps are out of sync. - init == C_NULL && error("Missing Julia object for global '$name'") - obj = Base.unsafe_pointer_to_objref(init) - if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) - val, hdr = materialize_box!(mod, gv, obj, init) - initializer!(gv, val) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - refs.embedded_pointer |= hdr >= UInt(64 << 4) - else - host_reference_slot_size(mod, gv, name) - refs.slots[name] = JuliaValueRef(obj) - end - end - return refs -end - -host_reference_word_type() = LLVM.IntType(8sizeof(UInt)) - -function host_reference_slot_size(mod::LLVM.Module, gv::GlobalVariable, name::String) - size = abi_size(datalayout(mod), global_value_type(gv)) - size == sizeof(UInt) || - error("Host reference slot '$name' has size $size, expected $(sizeof(UInt))") - return -end - -function host_reference_slot_initializer(gv::GlobalVariable, value::UInt) - T = global_value_type(gv) - if T isa LLVM.PointerType - return const_inttoptr(ConstantInt(UInt64(value)), T) - elseif T isa LLVM.IntegerType && width(T) == 8sizeof(UInt) - return ConstantInt(T, value) - end - error("Host reference slot '$(LLVM.name(gv))' has unsupported LLVM type $T") -end - -function check_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) - mod_gvs = globals(mod) - for name in keys(refs.slots) - haskey(mod_gvs, name) || error("Missing host reference slot '$name'") - end - return -end - -function prune_dead_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) - mod_gvs = globals(mod) - for name in collect(keys(refs.slots)) - haskey(mod_gvs, name) || delete!(refs.slots, name) - end - return -end - -function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) - check_host_reference_slots!(mod, refs) - mod_gvs = globals(mod) - for (name, ref) in refs.slots - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) - value = resolve_host_reference(ref) - val = host_reference_slot_initializer(gv, value) - initializer!(gv, val) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - constant!(gv, true) - refs.embedded_pointer = true - end - empty!(refs.slots) - return -end - -""" - emit_host_reference_definitions!(mod, refs) - -Emit host-reference slots as writable, null-initialized definitions. The loader must patch each -definition after loading the object. This requires a per-object symbol namespace. -""" -function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences) - check_host_reference_slots!(mod, refs) - mod_gvs = globals(mod) - slots = GlobalVariable[] - for name in keys(refs.slots) - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) - initializer!(gv, null(global_value_type(gv))) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, true) - push!(slots, gv) - end - isempty(slots) || set_used!(mod, slots...) - return -end - -""" - emit_host_reference_declarations!(mod, refs) - -Prepare host references for a loader that defines symbols before loading an object. - -Runtime-global slots are restored to direct references to their named libjulia globals. Julia -value slots remain external word-sized declarations; the loader must define each symbol to -point at a cell containing [`resolve_host_reference`](@ref) and keep the cell and referenced -value alive while the code is executable. - -Slot names are unique only within one compilation. A shared JIT namespace must uniquify them -or use a separate namespace for each object. -""" -function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) - check_host_reference_slots!(mod, refs) - mod_gvs = globals(mod) - for (name, ref) in collect(refs.slots) - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) - - if ref isa CGlobalRef - symbol = String(ref.symbol) - if symbol == name - initializer!(gv, nothing) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, false) - else - replacement = if haskey(globals(mod), symbol) - globals(mod)[symbol] - elseif haskey(functions(mod), symbol) - functions(mod)[symbol] - else - GlobalVariable(mod, host_reference_word_type(), symbol) - end - replacement = value_type(replacement) == value_type(gv) ? replacement : - const_pointercast(replacement, value_type(gv)) - replace_uses!(gv, replacement) - erase!(gv) - end - delete!(refs.slots, name) - else - isdeclaration(gv) || - error("Julia value host reference slot '$name' must be a declaration") - linkage!(gv, LLVM.API.LLVMExternalLinkage) - end - end - return -end - -function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, - src_mod::LLVM.Module, src_refs::HostReferences; - only_needed=false) - # Link-time optimization may already have removed an unreferenced global. This is the - # last point where its absence is provably dead rather than a broken relocation. - prune_dead_host_reference_slots!(dest_mod, dest_refs) - prune_dead_host_reference_slots!(src_mod, src_refs) - src_gvs = globals(src_mod) - dest_names = Set{String}(LLVM.name(v) for v in [collect(globals(dest_mod)); - collect(functions(dest_mod))]) - src_names = Set{String}(LLVM.name(v) for v in [collect(globals(src_mod)); - collect(functions(src_mod))]) - - function fresh_slot_name(name) - base = safe_name(name) * "_gpucompiler" - candidate = base - suffix = 0 - while candidate in dest_names || candidate in src_names - suffix += 1 - candidate = base * "_" * string(suffix) - end - return candidate - end - - for (name, ref) in collect(src_refs.slots) - dest_ref = get(dest_refs.slots, name, nothing) - haskey(src_gvs, name) || error("Missing source host reference slot '$name'") - if dest_ref !== nothing && same_host_reference(dest_ref, ref) - haskey(globals(dest_mod), name) || - error("Missing destination host reference slot '$name'") - continue - end - - gv = src_gvs[name] - if name in dest_names || dest_ref !== nothing - delete!(src_names, name) - LLVM.name!(gv, fresh_slot_name(name)) - push!(src_names, LLVM.name(gv)) - else - continue - end - actual_name = LLVM.name(gv) - delete!(src_refs.slots, name) - haskey(src_refs.slots, actual_name) && - error("Duplicate source host reference slot '$actual_name'") - src_refs.slots[actual_name] = ref - end - - link!(dest_mod, src_mod; only_needed) - for (name, ref) in src_refs.slots - if !haskey(globals(dest_mod), name) - only_needed && continue - error("Linked host reference slot '$name' is missing") - end - dest_ref = get(dest_refs.slots, name, nothing) - dest_ref === nothing && (dest_refs.slots[name] = ref) - dest_ref === nothing || same_host_reference(dest_ref, ref) || - error("Conflicting linked host reference slot '$name'") - end - for name in keys(dest_refs.slots) - haskey(globals(dest_mod), name) || - error("Merged host reference slot '$name' is missing") - end - dest_refs.embedded_pointer |= src_refs.embedded_pointer - return -end - -function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - refs = classify_gvs!(mod, gv_to_value) - refs.embedded_pointer |= !materialize_bool_singletons!(mod) - resolve_host_reference_slots!(mod, refs) - return !refs.embedded_pointer -end - -# Bool JuliaVariables are absent from `gv_to_value`; define one device box per name. -function materialize_bool_singletons!(mod::LLVM.Module) - portable = true - mod_gvs = globals(mod) - for (name, obj) in ("jl_true" => true, "jl_false" => false) - haskey(mod_gvs, name) || continue - gv = mod_gvs[name] - cur = initializer(gv) - if !(cur === nothing || LLVM.isnull(cur)) - # Existing definitions may contain session-specific addresses. - portable = false - continue - end - - init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) - val, hdr = materialize_box!(mod, gv, obj, init) - initializer!(gv, val) - constant!(gv, true) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - - # Stay conservative if Bool stops using a smalltag. - portable &= hdr < UInt(64 << 4) # jl_max_tags << 4 - end - return portable -end - -# emit a device-resident constant replica of the box holding `obj`; returns -# the constant to store in the slot, and the (gcbits-masked) header word -function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(obj), - init::Ptr{Cvoid}) - @assert isbitstype(typeof(obj)) && sizeof(obj) > 0 - - W = sizeof(Int) - hdr, bytes = GC.@preserve obj begin - # the header word transparently yields the smalltag immediate for - # smalltag types and the host type pointer otherwise; drop the gcbits - hdr = unsafe_load(Ptr{UInt}(init - W)) & ~UInt(15) - bytes = [unsafe_load(Ptr{UInt8}(init), i) for i in 1:sizeof(obj)] - hdr, bytes - end - - T_word = LLVM.IntType(8W) - T_byte = LLVM.Int8Type() - fields = LLVM.Constant[ConstantInt(T_word, hdr), ConstantDataArray(T_byte, bytes)] - payload_idx = 1 - if Base.datatype_alignment(typeof(obj)) > W - # pad so the payload lands at a 16-byte offset (JL_HEAP_ALIGNMENT max) - pushfirst!(fields, ConstantDataArray(T_byte, zeros(UInt8, 16 - W))) - payload_idx = 2 - end - boxinit = ConstantStruct(fields) - boxty = value_type(boxinit) - - box = GlobalVariable(mod, boxty, safe_name(LLVM.name(gv)) * "_box") - initializer!(box, boxinit) - constant!(box, true) - linkage!(box, LLVM.API.LLVMPrivateLinkage) - alignment!(box, 16) - unnamed_addr!(box, true) - - idx(i) = ConstantInt(LLVM.Int32Type(), i) - payload = const_gep(boxty, box, LLVM.Constant[idx(0), idx(payload_idx)]) - slotty = global_value_type(gv) - val = value_type(payload) == slotty ? payload : const_addrspacecast(payload, slotty) - return val, hdr -end # partially revert JuliaLang/julia#49391 — see #527 @static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master diff --git a/src/mcgen.jl b/src/mcgen.jl index 7593f590..8659ec74 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -24,83 +24,6 @@ function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, return end -# some Julia code contains references to objects in the CPU run-time, -# without actually using the contents or functionality of those objects. -# -# prime example are type tags, which reference the address of the allocated type. -# since those references are ephemeral, we can't eagerly resolve and emit them in the IR, -# but at the same time the GPU can't resolve them at run-time. -# -# this pass performs that resolution at link time. -struct CollectRuntimeGlobalReferences - job::CompilerJob - refs::HostReferences -end -function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) - changed = false - - for f in [collect(functions(mod)); collect(globals(mod))] - fn = LLVM.name(f) - # Julia value globals are already represented by an identity in `refs`; they may - # use a `jl_*` spelling but are not exported libjulia runtime globals. - f isa LLVM.GlobalVariable && haskey(self.refs.slots, fn) && continue - if isdeclaration(f) && (!(f isa LLVM.Function) || !LLVM.isintrinsic(f)) && - startswith(fn, "jl_") - slot = nothing - function runtime_global_slot() - if slot === nothing - name = safe_name("gpu_" * fn) - slot = GlobalVariable(mod, host_reference_word_type(), name) - actual_name = LLVM.name(slot) - haskey(self.refs.slots, actual_name) && - error("Duplicate Julia runtime global slot '$actual_name'") - self.refs.slots[actual_name] = CGlobalRef(Symbol(fn)) - end - slot - end - - function replace_bindings!(value) - changed = false - for use in collect(uses(value)) - val = user(use) - if isa(val, LLVM.ConstantExpr) - # recurse - changed |= replace_bindings!(val) - elseif isa(val, LLVM.LoadInst) - T = value_type(val) - if !(T isa LLVM.PointerType || - (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) - error("Unsupported Julia runtime global '$fn' load of LLVM type $T") - end - @dispose builder=IRBuilder() begin - position!(builder, val) - replacement = load!(builder, host_reference_word_type(), - runtime_global_slot()) - if T isa LLVM.PointerType - replacement = inttoptr!(builder, replacement, T) - end - replace_uses!(val, replacement) - end - erase!(val) - changed = true - end - end - changed - end - - changed |= replace_bindings!(f) - end - end - - return changed -end -collect_runtime_global_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences) = - CollectRuntimeGlobalReferences(job, refs)(mod) -CollectRuntimeGlobalReferencesPass(job, refs=HostReferences()) = - NewPMModulePass("CollectRuntimeGlobalReferences", CollectRuntimeGlobalReferences(job, refs)) - - function mcgen(@nospecialize(job::CompilerJob), mod::LLVM.Module, format=LLVM.API.LLVMAssemblyFile) tm = llvm_machine(job.config.target) diff --git a/src/relocation.jl b/src/relocation.jl new file mode 100644 index 00000000..f41d0c63 --- /dev/null +++ b/src/relocation.jl @@ -0,0 +1,494 @@ +# Host-reference relocation lifecycle +# +# produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) +# +# Julia value globals and libjulia runtime globals are represented by symbolic slots until +# the backend lowers them. Slot names identify their HostReference and, once assigned, must +# remain globally unique and stable through linking. +# +# This mirrors Julia's own mechanisms: codegen's identity-keyed global_targets slots, the +# sysimage jl_gvars table patched by jl_update_all_gvars, and JIT absoluteSymbols definitions. + +""" + JuliaValueRef(value) + +A Julia value with a stable address (a heap object, symbol, or singleton), used as the +serializable identity of a host reference. Resolve it in the active session with +[`resolve_host_reference`](@ref); a loader that writes the resulting address into device +storage must keep `value` rooted for as long as that storage remains reachable. +""" +struct JuliaValueRef + value::Any + + function JuliaValueRef(value) + isbitstype(typeof(value)) && sizeof(value) > 0 && + error("JuliaValueRef requires an object with a stable address") + new(value) + end +end + +""" + CGlobalRef(symbol) + +A named libjulia C data global. Resolution returns the word stored in that global in the +current Julia process. +""" +struct CGlobalRef + symbol::Symbol +end + +""" + HostReference + +A serializable source for a host-derived word: either a [`JuliaValueRef`](@ref) or a +[`CGlobalRef`](@ref). +""" +const HostReference = Union{JuliaValueRef,CGlobalRef} + +""" + HostReferences(slots, embedded_pointer) + +Host-reference metadata accompanying a module. `slots` maps each object-code symbol to the +source of the word that must be written there. `embedded_pointer` conservatively records that +an unavoidable session pointer has been embedded, so the artifact cannot be serialized into a +package image even if `slots` is empty. +""" +mutable struct HostReferences + slots::Dict{String,HostReference} + embedded_pointer::Bool +end + +HostReferences() = HostReferences(Dict{String,HostReference}(), false) +copy_host_references(refs::HostReferences) = + HostReferences(copy(refs.slots), refs.embedded_pointer) + +same_host_reference(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value +same_host_reference(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol +same_host_reference(::HostReference, ::HostReference) = false + +""" + resolve_host_reference(ref) -> UInt + +Resolve a host reference to its word in the current Julia process. +""" +function resolve_host_reference(ref::JuliaValueRef) + box = Any[ref.value] + GC.@preserve box begin + return unsafe_load(Base.unsafe_convert(Ptr{UInt}, pointer(box))) + end +end +function resolve_host_reference(ref::CGlobalRef) + address = ccall(:jl_cglobal, Any, (Any, Any), String(ref.symbol), UInt) + return unsafe_load(address) +end + +""" + lower_host_references!(job, mod, refs) + +Backend hook for lowering live host references before object emission. + +The default implementation resolves them in the current Julia process, making the result +session-dependent. Loaders may instead emit module-owned definitions that are patched after +loading, or external declarations that are defined before loading. +""" +function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences) + resolve_host_reference_slots!(mod, refs) +end + + +function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + refs = HostReferences() + mod_gvs = globals(mod) + for (name, init) in gv_to_value + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + cur = initializer(gv) + if !(cur === nothing || LLVM.isnull(cur)) + refs.embedded_pointer = true + continue + end + + # jl_get_llvm_gvs and jl_get_llvm_gv_inits report an initializer for every + # mapped global, so a null here means those maps are out of sync. + init == C_NULL && error("Missing Julia object for global '$name'") + obj = Base.unsafe_pointer_to_objref(init) + if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) + val, hdr = materialize_box!(mod, gv, obj, init) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + refs.embedded_pointer |= hdr >= UInt(64 << 4) + else + host_reference_slot_size(mod, gv, name) + refs.slots[name] = JuliaValueRef(obj) + end + end + return refs +end + +host_reference_word_type() = LLVM.IntType(8sizeof(UInt)) + +function host_reference_slot_size(mod::LLVM.Module, gv::GlobalVariable, name::String) + size = abi_size(datalayout(mod), global_value_type(gv)) + size == sizeof(UInt) || + error("Host reference slot '$name' has size $size, expected $(sizeof(UInt))") + return +end + +function host_reference_slot_initializer(gv::GlobalVariable, value::UInt) + T = global_value_type(gv) + if T isa LLVM.PointerType + return const_inttoptr(ConstantInt(UInt64(value)), T) + elseif T isa LLVM.IntegerType && width(T) == 8sizeof(UInt) + return ConstantInt(T, value) + end + error("Host reference slot '$(LLVM.name(gv))' has unsupported LLVM type $T") +end + +function check_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + mod_gvs = globals(mod) + for name in keys(refs.slots) + haskey(mod_gvs, name) || error("Missing host reference slot '$name'") + end + return +end + +function prune_dead_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + mod_gvs = globals(mod) + for name in collect(keys(refs.slots)) + haskey(mod_gvs, name) || delete!(refs.slots, name) + end + return +end + +function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) + check_host_reference_slots!(mod, refs) + mod_gvs = globals(mod) + for (name, ref) in refs.slots + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + value = resolve_host_reference(ref) + val = host_reference_slot_initializer(gv, value) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + constant!(gv, true) + refs.embedded_pointer = true + end + empty!(refs.slots) + return +end + +""" + emit_host_reference_definitions!(mod, refs) + +Emit host-reference slots as writable, null-initialized definitions. The loader must patch each +definition after loading the object. This requires a per-object symbol namespace. +""" +function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences) + check_host_reference_slots!(mod, refs) + mod_gvs = globals(mod) + slots = GlobalVariable[] + for name in keys(refs.slots) + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + initializer!(gv, null(global_value_type(gv))) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, true) + push!(slots, gv) + end + isempty(slots) || set_used!(mod, slots...) + return +end + +""" + emit_host_reference_declarations!(mod, refs) + +Prepare host references for a loader that defines symbols before loading an object. + +Runtime-global slots are restored to direct references to their named libjulia globals. Julia +value slots remain external word-sized declarations; the loader must define each symbol to +point at a cell containing [`resolve_host_reference`](@ref) and keep the cell and referenced +value alive while the code is executable. + +Slot names are unique only within one compilation. A shared JIT namespace must uniquify them +or use a separate namespace for each object. +""" +function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) + check_host_reference_slots!(mod, refs) + mod_gvs = globals(mod) + for (name, ref) in collect(refs.slots) + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + + if ref isa CGlobalRef + symbol = String(ref.symbol) + if symbol == name + initializer!(gv, nothing) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + else + replacement = if haskey(globals(mod), symbol) + globals(mod)[symbol] + elseif haskey(functions(mod), symbol) + functions(mod)[symbol] + else + GlobalVariable(mod, host_reference_word_type(), symbol) + end + replacement = value_type(replacement) == value_type(gv) ? replacement : + const_pointercast(replacement, value_type(gv)) + replace_uses!(gv, replacement) + erase!(gv) + end + delete!(refs.slots, name) + else + isdeclaration(gv) || + error("Julia value host reference slot '$name' must be a declaration") + linkage!(gv, LLVM.API.LLVMExternalLinkage) + end + end + return +end + +function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, + src_mod::LLVM.Module, src_refs::HostReferences; + only_needed=false) + # Link-time optimization may already have removed an unreferenced global. This is the + # last point where its absence is provably dead rather than a broken relocation. + prune_dead_host_reference_slots!(dest_mod, dest_refs) + prune_dead_host_reference_slots!(src_mod, src_refs) + src_gvs = globals(src_mod) + dest_names = Set{String}(LLVM.name(v) for v in [collect(globals(dest_mod)); + collect(functions(dest_mod))]) + src_names = Set{String}(LLVM.name(v) for v in [collect(globals(src_mod)); + collect(functions(src_mod))]) + + function fresh_slot_name(name) + base = safe_name(name) * "_gpucompiler" + candidate = base + suffix = 0 + while candidate in dest_names || candidate in src_names + suffix += 1 + candidate = base * "_" * string(suffix) + end + return candidate + end + + for (name, ref) in collect(src_refs.slots) + dest_ref = get(dest_refs.slots, name, nothing) + haskey(src_gvs, name) || error("Missing source host reference slot '$name'") + if dest_ref !== nothing && same_host_reference(dest_ref, ref) + haskey(globals(dest_mod), name) || + error("Missing destination host reference slot '$name'") + continue + end + + gv = src_gvs[name] + if name in dest_names || dest_ref !== nothing + delete!(src_names, name) + LLVM.name!(gv, fresh_slot_name(name)) + push!(src_names, LLVM.name(gv)) + else + continue + end + actual_name = LLVM.name(gv) + delete!(src_refs.slots, name) + haskey(src_refs.slots, actual_name) && + error("Duplicate source host reference slot '$actual_name'") + src_refs.slots[actual_name] = ref + end + + link!(dest_mod, src_mod; only_needed) + for (name, ref) in src_refs.slots + if !haskey(globals(dest_mod), name) + only_needed && continue + error("Linked host reference slot '$name' is missing") + end + dest_ref = get(dest_refs.slots, name, nothing) + dest_ref === nothing && (dest_refs.slots[name] = ref) + dest_ref === nothing || same_host_reference(dest_ref, ref) || + error("Conflicting linked host reference slot '$name'") + end + for name in keys(dest_refs.slots) + haskey(globals(dest_mod), name) || + error("Merged host reference slot '$name' is missing") + end + dest_refs.embedded_pointer |= src_refs.embedded_pointer + return +end + +function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) + refs = classify_gvs!(mod, gv_to_value) + refs.embedded_pointer |= !materialize_bool_singletons!(mod) + resolve_host_reference_slots!(mod, refs) + return !refs.embedded_pointer +end + +# Bool JuliaVariables are absent from `gv_to_value`; define one device box per name. +function materialize_bool_singletons!(mod::LLVM.Module) + portable = true + mod_gvs = globals(mod) + for (name, obj) in ("jl_true" => true, "jl_false" => false) + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + cur = initializer(gv) + if !(cur === nothing || LLVM.isnull(cur)) + # Existing definitions may contain session-specific addresses. + portable = false + continue + end + + init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) + val, hdr = materialize_box!(mod, gv, obj, init) + initializer!(gv, val) + constant!(gv, true) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + + # Stay conservative if Bool stops using a smalltag. + portable &= hdr < UInt(64 << 4) # jl_max_tags << 4 + end + return portable +end + +# emit a device-resident constant replica of the box holding `obj`; returns +# the constant to store in the slot, and the (gcbits-masked) header word +function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(obj), + init::Ptr{Cvoid}) + @assert isbitstype(typeof(obj)) && sizeof(obj) > 0 + + W = sizeof(Int) + hdr, bytes = GC.@preserve obj begin + # the header word transparently yields the smalltag immediate for + # smalltag types and the host type pointer otherwise; drop the gcbits + hdr = unsafe_load(Ptr{UInt}(init - W)) & ~UInt(15) + bytes = [unsafe_load(Ptr{UInt8}(init), i) for i in 1:sizeof(obj)] + hdr, bytes + end + + T_word = LLVM.IntType(8W) + T_byte = LLVM.Int8Type() + fields = LLVM.Constant[ConstantInt(T_word, hdr), ConstantDataArray(T_byte, bytes)] + payload_idx = 1 + if Base.datatype_alignment(typeof(obj)) > W + # pad so the payload lands at a 16-byte offset (JL_HEAP_ALIGNMENT max) + pushfirst!(fields, ConstantDataArray(T_byte, zeros(UInt8, 16 - W))) + payload_idx = 2 + end + boxinit = ConstantStruct(fields) + boxty = value_type(boxinit) + + box = GlobalVariable(mod, boxty, safe_name(LLVM.name(gv)) * "_box") + initializer!(box, boxinit) + constant!(box, true) + linkage!(box, LLVM.API.LLVMPrivateLinkage) + alignment!(box, 16) + unnamed_addr!(box, true) + + idx(i) = ConstantInt(LLVM.Int32Type(), i) + payload = const_gep(boxty, box, LLVM.Constant[idx(0), idx(payload_idx)]) + slotty = global_value_type(gv) + val = value_type(payload) == slotty ? payload : const_addrspacecast(payload, slotty) + return val, hdr +end + + +# some Julia code contains references to objects in the CPU run-time, +# without actually using the contents or functionality of those objects. +# +# prime example are type tags, which reference the address of the allocated type. +# since those references are ephemeral, we can't eagerly resolve and emit them in the IR, +# but at the same time the GPU can't resolve them at run-time. +# +# this pass performs that resolution at link time. +struct CollectRuntimeGlobalReferences + job::CompilerJob + refs::HostReferences +end +function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) + changed = false + + for f in [collect(functions(mod)); collect(globals(mod))] + fn = LLVM.name(f) + # Julia value globals are already represented by an identity in `refs`; they may + # use a `jl_*` spelling but are not exported libjulia runtime globals. + f isa LLVM.GlobalVariable && haskey(self.refs.slots, fn) && continue + if isdeclaration(f) && (!(f isa LLVM.Function) || !LLVM.isintrinsic(f)) && + startswith(fn, "jl_") + slot = nothing + function runtime_global_slot() + if slot === nothing + name = safe_name("gpu_" * fn) + slot = GlobalVariable(mod, host_reference_word_type(), name) + actual_name = LLVM.name(slot) + haskey(self.refs.slots, actual_name) && + error("Duplicate Julia runtime global slot '$actual_name'") + self.refs.slots[actual_name] = CGlobalRef(Symbol(fn)) + end + slot + end + + function replace_bindings!(value) + changed = false + for use in collect(uses(value)) + val = user(use) + if isa(val, LLVM.ConstantExpr) + # recurse + changed |= replace_bindings!(val) + elseif isa(val, LLVM.LoadInst) + T = value_type(val) + if !(T isa LLVM.PointerType || + (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) + error("Unsupported Julia runtime global '$fn' load of LLVM type $T") + end + @dispose builder=IRBuilder() begin + position!(builder, val) + replacement = load!(builder, host_reference_word_type(), + runtime_global_slot()) + if T isa LLVM.PointerType + replacement = inttoptr!(builder, replacement, T) + end + replace_uses!(val, replacement) + end + erase!(val) + changed = true + end + end + changed + end + + changed |= replace_bindings!(f) + end + end + + return changed +end +collect_runtime_global_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences) = + CollectRuntimeGlobalReferences(job, refs)(mod) +CollectRuntimeGlobalReferencesPass(job, refs=HostReferences()) = + NewPMModulePass("CollectRuntimeGlobalReferences", CollectRuntimeGlobalReferences(job, refs)) + + +function referenced_object(value, refs::HostReferences) + # This is best-effort: optimized shapes fall back to the unknown-binding error path. + while value isa ConstantExpr && + opcode(value) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) + value = first(operands(value)) + end + if value isa LLVM.LoadInst + source = first(operands(value)) + while source isa ConstantExpr && + opcode(source) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) + source = first(operands(source)) + end + if source isa GlobalVariable + ref = get(refs.slots, LLVM.name(source), nothing) + ref isa JuliaValueRef && return Some(ref.value) + end + elseif value isa ConstantExpr && opcode(value) == LLVM.API.LLVMIntToPtr + ptr = Ptr{Cvoid}(convert(Int, first(operands(value)))) + return Some(Base.unsafe_pointer_to_objref(ptr)) + end + return nothing +end diff --git a/src/validation.jl b/src/validation.jl index 9cd0514c..106e83c9 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -197,29 +197,6 @@ end const libjulia = Ref{Ptr{Cvoid}}(C_NULL) -function referenced_object(value, refs::HostReferences) - # This is best-effort: optimized shapes fall back to the unknown-binding error path. - while value isa ConstantExpr && - opcode(value) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) - value = first(operands(value)) - end - if value isa LLVM.LoadInst - source = first(operands(value)) - while source isa ConstantExpr && - opcode(source) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) - source = first(operands(source)) - end - if source isa GlobalVariable - ref = get(refs.slots, LLVM.name(source), nothing) - ref isa JuliaValueRef && return Some(ref.value) - end - elseif value isa ConstantExpr && opcode(value) == LLVM.API.LLVMIntToPtr - ptr = Ptr{Cvoid}(convert(Int, first(operands(value)))) - return Some(Base.unsafe_pointer_to_objref(ptr)) - end - return nothing -end - function check_ir!(job, errors::Vector{IRError}, inst::LLVM.LoadInst) bt = backtrace(inst) src = operands(inst)[1] From ddd86314bbf289db371750a18f459000d2f4d8fd Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 18:58:45 +0200 Subject: [PATCH 13/45] Assign host-reference slot identities at creation --- src/driver.jl | 4 +- src/irgen.jl | 22 +++----- src/relocation.jl | 130 +++++++++++++++------------------------------- src/rtlib.jl | 8 +-- test/native.jl | 29 +++-------- test/ptx.jl | 2 +- 6 files changed, 61 insertions(+), 134 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index c5a932a3..579a18db 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -180,7 +180,7 @@ const __llvm_initialized = Ref(false) end @tracepoint "IR generation" begin - ir, compiled, gv_to_value = irgen(job) + ir, compiled, host_references = irgen(job) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -192,7 +192,6 @@ const __llvm_initialized = Ref(false) # finalize the current module. this needs to happen before linking deferred modules, # since those modules have been finalized themselves, and we don't want to re-finalize. entry = finish_module!(job, ir, entry) - host_references = classify_gvs!(ir, gv_to_value) # deferred code generation has_deferred_jobs = job.config.toplevel && !job.config.only_entry && @@ -334,7 +333,6 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) - host_references.embedded_pointer |= !materialize_bool_singletons!(ir) host_references.embedded_pointer && mark_session_dependent!(job) if job.config.optimize diff --git a/src/irgen.jl b/src/irgen.jl index f7056d68..c5dc218f 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -2,6 +2,7 @@ function irgen(@nospecialize(job::CompilerJob)) mod, compiled, gv_to_value = @tracepoint "emission" compile_method_instance(job) + host_references = classify_gvs!(mod, gv_to_value) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -43,23 +44,12 @@ function irgen(@nospecialize(job::CompilerJob)) end end - # Julia value globals can be declarations. Sanitize every definition as well as every - # mapped global, but leave unrelated external declarations alone because their names are - # part of their ABI. + # sanitize defined globals (external declaration names are part of their ABI) for val in collect(globals(mod)) - old_name = LLVM.name(val) - mapped = haskey(gv_to_value, old_name) - isdeclaration(val) && !mapped && continue - new_name = safe_name(old_name) - if old_name != new_name + isdeclaration(val) && continue + new_name = safe_name(LLVM.name(val)) + if LLVM.name(val) != new_name LLVM.name!(val, new_name) - actual_name = LLVM.name(val) - if mapped - init = pop!(gv_to_value, old_name) - haskey(gv_to_value, actual_name) && - error("Duplicate Julia value global '$actual_name'") - gv_to_value[actual_name] = init - end end end @@ -160,7 +150,7 @@ function irgen(@nospecialize(job::CompilerJob)) lower_alloca!(job, mod) end - return mod, compiled, gv_to_value + return mod, compiled, host_references end diff --git a/src/relocation.jl b/src/relocation.jl index f41d0c63..577d39a2 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -3,8 +3,10 @@ # produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) # # Julia value globals and libjulia runtime globals are represented by symbolic slots until -# the backend lowers them. Slot names identify their HostReference and, once assigned, must -# remain globally unique and stable through linking. +# the backend lowers them. A slot's LLVM name is globally unique and assigned exactly once: +# the readable Julia name plus object identity for JuliaValueRef slots, and `gpu_` plus the +# C symbol for CGlobalRef slots. Same name therefore means the same HostReference, so linking +# can merge declarations and metadata without renaming either one. # # This mirrors Julia's own mechanisms: codegen's identity-keyed global_targets slots, the # sysimage jl_gvars table patched by jl_update_all_gvars, and JIT absoluteSymbols definitions. @@ -59,8 +61,6 @@ mutable struct HostReferences end HostReferences() = HostReferences(Dict{String,HostReference}(), false) -copy_host_references(refs::HostReferences) = - HostReferences(copy(refs.slots), refs.embedded_pointer) same_host_reference(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value same_host_reference(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol @@ -120,9 +120,38 @@ function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) refs.embedded_pointer |= hdr >= UInt(64 << 4) else host_reference_slot_size(mod, gv, name) - refs.slots[name] = JuliaValueRef(obj) + slot_name = safe_name(name) * "_" * string(objectid(obj); base=16) + if slot_name != name && + (haskey(globals(mod), slot_name) || haskey(functions(mod), slot_name)) + error("Host reference slot name '$slot_name' is already in use") + end + LLVM.name!(gv, slot_name) + LLVM.name(gv) == slot_name || + error("Host reference slot name '$slot_name' is already in use") + refs.slots[slot_name] = JuliaValueRef(obj) end end + + # Bool JuliaVariables are absent from `gv_to_value`; define one device box per module. + for (name, obj) in ("jl_true" => true, "jl_false" => false) + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + cur = initializer(gv) + if !(cur === nothing || LLVM.isnull(cur)) + # Existing definitions may contain session-specific addresses. + refs.embedded_pointer = true + continue + end + + init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) + val, hdr = materialize_box!(mod, gv, obj, init) + initializer!(gv, val) + constant!(gv, true) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + + # Stay conservative if Bool stops using a smalltag. + refs.embedded_pointer |= hdr >= UInt(64 << 4) # jl_max_tags << 4 + end return refs end @@ -254,65 +283,15 @@ end function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, src_mod::LLVM.Module, src_refs::HostReferences; only_needed=false) - # Link-time optimization may already have removed an unreferenced global. This is the - # last point where its absence is provably dead rather than a broken relocation. - prune_dead_host_reference_slots!(dest_mod, dest_refs) - prune_dead_host_reference_slots!(src_mod, src_refs) - src_gvs = globals(src_mod) - dest_names = Set{String}(LLVM.name(v) for v in [collect(globals(dest_mod)); - collect(functions(dest_mod))]) - src_names = Set{String}(LLVM.name(v) for v in [collect(globals(src_mod)); - collect(functions(src_mod))]) - - function fresh_slot_name(name) - base = safe_name(name) * "_gpucompiler" - candidate = base - suffix = 0 - while candidate in dest_names || candidate in src_names - suffix += 1 - candidate = base * "_" * string(suffix) - end - return candidate - end - - for (name, ref) in collect(src_refs.slots) - dest_ref = get(dest_refs.slots, name, nothing) - haskey(src_gvs, name) || error("Missing source host reference slot '$name'") - if dest_ref !== nothing && same_host_reference(dest_ref, ref) - haskey(globals(dest_mod), name) || - error("Missing destination host reference slot '$name'") - continue - end - - gv = src_gvs[name] - if name in dest_names || dest_ref !== nothing - delete!(src_names, name) - LLVM.name!(gv, fresh_slot_name(name)) - push!(src_names, LLVM.name(gv)) - else - continue - end - actual_name = LLVM.name(gv) - delete!(src_refs.slots, name) - haskey(src_refs.slots, actual_name) && - error("Duplicate source host reference slot '$actual_name'") - src_refs.slots[actual_name] = ref - end - link!(dest_mod, src_mod; only_needed) for (name, ref) in src_refs.slots - if !haskey(globals(dest_mod), name) - only_needed && continue - error("Linked host reference slot '$name' is missing") - end - dest_ref = get(dest_refs.slots, name, nothing) - dest_ref === nothing && (dest_refs.slots[name] = ref) - dest_ref === nothing || same_host_reference(dest_ref, ref) || - error("Conflicting linked host reference slot '$name'") - end - for name in keys(dest_refs.slots) - haskey(globals(dest_mod), name) || - error("Merged host reference slot '$name' is missing") + # A slot absent from the linked module was dead (DCE'd or not imported under + # `only_needed`); its relocation dies with it. + haskey(globals(dest_mod), name) || continue + existing = get(dest_refs.slots, name, nothing) + existing === nothing || same_host_reference(existing, ref) || + error("Host reference slot '$name' refers to conflicting values") + dest_refs.slots[name] = ref end dest_refs.embedded_pointer |= src_refs.embedded_pointer return @@ -320,37 +299,10 @@ end function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) refs = classify_gvs!(mod, gv_to_value) - refs.embedded_pointer |= !materialize_bool_singletons!(mod) resolve_host_reference_slots!(mod, refs) return !refs.embedded_pointer end -# Bool JuliaVariables are absent from `gv_to_value`; define one device box per name. -function materialize_bool_singletons!(mod::LLVM.Module) - portable = true - mod_gvs = globals(mod) - for (name, obj) in ("jl_true" => true, "jl_false" => false) - haskey(mod_gvs, name) || continue - gv = mod_gvs[name] - cur = initializer(gv) - if !(cur === nothing || LLVM.isnull(cur)) - # Existing definitions may contain session-specific addresses. - portable = false - continue - end - - init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) - val, hdr = materialize_box!(mod, gv, obj, init) - initializer!(gv, val) - constant!(gv, true) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - - # Stay conservative if Bool stops using a smalltag. - portable &= hdr < UInt(64 << 4) # jl_max_tags << 4 - end - return portable -end - # emit a device-resident constant replica of the box holding `obj`; returns # the constant to store in the slot, and the (gcbits-masked) header word function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(obj), diff --git a/src/rtlib.jl b/src/rtlib.jl index 84c1aa3a..d5baa043 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -84,9 +84,9 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, # inference itself. ci, res = runtime_function_results(rt_job) if res !== nothing && res.bitcode !== nothing - cached_refs = copy_host_references(res.host_references) link_with_host_references!(mod, refs, - parse(LLVM.Module, MemoryBuffer(res.bitcode)), cached_refs) + parse(LLVM.Module, MemoryBuffer(res.bitcode)), + res.host_references) ci === nothing && (ci = runtime_code_instance(rt_job)) return ci::CodeInstance end @@ -119,7 +119,7 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, ci === nothing && (ci = runtime_code_instance(rt_job)) res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) res.bitcode = take!(io) - res.host_references = copy_host_references(meta.host_references) + res.host_references = meta.host_references link_with_host_references!(mod, refs, new_mod, meta.host_references) return ci::CodeInstance @@ -248,5 +248,5 @@ const runtime_libs_lock = ReentrantLock() end return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true), - copy_host_references(cached.host_references) + cached.host_references end diff --git a/test/native.jl b/test/native.jl index 0b8eac02..487d7ac2 100644 --- a/test/native.jl +++ b/test/native.jl @@ -866,27 +866,6 @@ end refs(name, value) = GPUCompiler.HostReferences(Dict(name => GPUCompiler.JuliaValueRef(value)), false) - # Three unrelated values requested the same slot name. Each load keeps its own - # relocation after linking, including the second fresh-name suffix. - dest = slot_module("slot", "first") - dest_refs = refs("slot", :first) - for (entry, value) in (("second", :second), ("third", :third)) - src = slot_module("slot", entry) - src_refs = refs("slot", value) - GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) - end - @test Set(keys(dest_refs.slots)) == - Set(("slot", "slot_gpucompiler", "slot_gpucompiler_1")) - @test Set(ref.value for ref in values(dest_refs.slots)) == Set((:first, :second, :third)) - - # A global outside the metadata table is still part of LLVM's namespace. - dest = slot_module("slot", "occupied") - dest_refs = GPUCompiler.HostReferences() - src = slot_module("slot", "source") - src_refs = refs("slot", :source) - GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) - @test only(keys(dest_refs.slots)) == "slot_gpucompiler" - # Equal Julia identities deliberately share a single slot. dest = slot_module("slot", "first") dest_refs = refs("slot", :shared) @@ -896,6 +875,14 @@ end @test collect(keys(dest_refs.slots)) == ["slot"] @test occursin("@slot = external global i64", string(dest)) + # The name is the slot identity, so conflicting metadata is an error. + dest = slot_module("slot", "first") + dest_refs = refs("slot", :first) + src = slot_module("slot", "second") + src_refs = refs("slot", :second) + @test_throws ErrorException GPUCompiler.link_with_host_references!( + dest, dest_refs, src, src_refs) + # `only_needed` must keep metadata for imported slots and discard metadata for # source globals that the LLVM linker did not import. dest = parse(LLVM.Module, """ diff --git a/test/ptx.jl b/test/ptx.jl index 5a5307e3..aaf1190e 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -30,7 +30,7 @@ end @testset "global variable relocation" begin # references to Julia objects (`julia.constgv` globals, e.g. Symbol literals) must - # survive until `relocate_gvs!` bakes in their addresses at the toplevel link step. + # survive as host-reference slots until backend lowering at object emission. # they used to be kept alive as internal globals with a null initializer, which the # GlobalOpt run in `finish_module!` folded away, constant-folding any comparison # against them (JuliaGPU/CUDA.jl#3185: kernels specialized on Symbols misbehaved). From 843efb2404c01e1f37bdc702739933ab1c6fa051 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:00:21 +0200 Subject: [PATCH 14/45] Run host-reference lowering once at object emission --- src/mcgen.jl | 26 +++++++++++++++++--------- src/relocation.jl | 36 +++++++++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 16 deletions(-) diff --git a/src/mcgen.jl b/src/mcgen.jl index 8659ec74..6b1bdd48 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -2,25 +2,33 @@ # final preparations for the module to be compiled to machine code # these passes should not be run when e.g. compiling to write to disk. -function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences=HostReferences()) - prune_dead_host_reference_slots!(mod, refs) - collect_runtime_global_references!(job, mod, refs) - lower_host_references!(job, mod, refs) +function run_cleanup_pipeline!(@nospecialize(job::CompilerJob), mod::LLVM.Module) @dispose pb=NewPMPassBuilder() begin - register!(pb, CollectRuntimeGlobalReferencesPass(job, refs)) - add!(pb, RecomputeGlobalsAAPass()) add!(pb, GlobalOptPass()) - add!(pb, CollectRuntimeGlobalReferencesPass(job, refs)) add!(pb, GlobalDCEPass()) add!(pb, StripDeadPrototypesPass()) - run!(pb, mod, llvm_machine(job.config.target)) end + return +end +function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + refs::HostReferences=HostReferences()) + # Clean up first so only live references get slots and get lowered. + run_cleanup_pipeline!(job, mod) prune_dead_host_reference_slots!(mod, refs) + + collect_runtime_global_references!(job, mod, refs) lower_host_references!(job, mod, refs) + + # Fold constants exposed by eager lowering, and discard slots made dead by either + # lowering strategy. + run_cleanup_pipeline!(job, mod) + prune_dead_host_reference_slots!(mod, refs) + + has_unresolved_runtime_global_loads(mod, refs) && + error("Unresolved Julia runtime global load after host-reference lowering") return end diff --git a/src/relocation.jl b/src/relocation.jl index 577d39a2..1be3dce0 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -370,12 +370,15 @@ function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) slot = nothing function runtime_global_slot() if slot === nothing - name = safe_name("gpu_" * fn) + name = "gpu_" * fn + (haskey(globals(mod), name) || haskey(functions(mod), name)) && + error("Julia runtime global slot name '$name' is already in use") slot = GlobalVariable(mod, host_reference_word_type(), name) - actual_name = LLVM.name(slot) - haskey(self.refs.slots, actual_name) && - error("Duplicate Julia runtime global slot '$actual_name'") - self.refs.slots[actual_name] = CGlobalRef(Symbol(fn)) + LLVM.name(slot) == name || + error("Julia runtime global slot name '$name' is already in use") + haskey(self.refs.slots, name) && + error("Duplicate Julia runtime global slot '$name'") + self.refs.slots[name] = CGlobalRef(Symbol(fn)) end slot end @@ -418,8 +421,27 @@ end collect_runtime_global_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, refs::HostReferences) = CollectRuntimeGlobalReferences(job, refs)(mod) -CollectRuntimeGlobalReferencesPass(job, refs=HostReferences()) = - NewPMModulePass("CollectRuntimeGlobalReferences", CollectRuntimeGlobalReferences(job, refs)) + +function has_unresolved_runtime_global_loads(mod::LLVM.Module, refs::HostReferences) + function has_load(value) + for use in uses(value) + val = user(use) + val isa LLVM.LoadInst && return true + val isa LLVM.ConstantExpr && has_load(val) && return true + end + return false + end + + for value in [collect(functions(mod)); collect(globals(mod))] + name = LLVM.name(value) + value isa LLVM.GlobalVariable && haskey(refs.slots, name) && continue + isdeclaration(value) || continue + value isa LLVM.Function && LLVM.isintrinsic(value) && continue + startswith(name, "jl_") || continue + has_load(value) && return true + end + return false +end function referenced_object(value, refs::HostReferences) From 3fe49cb6c866d0a61c2817f42a0c4e456f3b1d64 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:01:39 +0200 Subject: [PATCH 15/45] Keep declaration-lowered host references uniform --- src/relocation.jl | 43 ++++++++----------------------------------- test/native.jl | 12 ++++++------ 2 files changed, 14 insertions(+), 41 deletions(-) diff --git a/src/relocation.jl b/src/relocation.jl index 1be3dce0..79b2d075 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -235,47 +235,20 @@ end Prepare host references for a loader that defines symbols before loading an object. -Runtime-global slots are restored to direct references to their named libjulia globals. Julia -value slots remain external word-sized declarations; the loader must define each symbol to -point at a cell containing [`resolve_host_reference`](@ref) and keep the cell and referenced -value alive while the code is executable. - -Slot names are unique only within one compilation. A shared JIT namespace must uniquify them -or use a separate namespace for each object. +Every slot remains an external word-sized declaration. The loader must define each symbol to +point at a cell containing [`resolve_host_reference`](@ref), and keep the cell and any +referenced Julia value alive while the code is executable. """ function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) check_host_reference_slots!(mod, refs) mod_gvs = globals(mod) - for (name, ref) in collect(refs.slots) + for name in keys(refs.slots) gv = mod_gvs[name] host_reference_slot_size(mod, gv, name) - - if ref isa CGlobalRef - symbol = String(ref.symbol) - if symbol == name - initializer!(gv, nothing) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, false) - else - replacement = if haskey(globals(mod), symbol) - globals(mod)[symbol] - elseif haskey(functions(mod), symbol) - functions(mod)[symbol] - else - GlobalVariable(mod, host_reference_word_type(), symbol) - end - replacement = value_type(replacement) == value_type(gv) ? replacement : - const_pointercast(replacement, value_type(gv)) - replace_uses!(gv, replacement) - erase!(gv) - end - delete!(refs.slots, name) - else - isdeclaration(gv) || - error("Julia value host reference slot '$name' must be a declaration") - linkage!(gv, LLVM.API.LLVMExternalLinkage) - end + isdeclaration(gv) || error("Host reference slot '$name' must be a declaration") + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) end return end diff --git a/test/native.jl b/test/native.jl index 487d7ac2..5671f623 100644 --- a/test/native.jl +++ b/test/native.jl @@ -828,9 +828,9 @@ end @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) GPUCompiler.emit_host_reference_declarations!(mod, refs) - @test isempty(refs.slots) - @test !haskey(globals(mod), name) - @test occursin("load i64, $(function_word_ptr("jl_float32_type"))", string(mod)) + @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test isdeclaration(globals(mod)[name]) + @test occursin("@$name = external global i64", string(mod)) mod = parse(LLVM.Module, """ @jl_float32_type = external global $word_ptr @@ -843,9 +843,9 @@ end @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) name = only(keys(refs.slots)) GPUCompiler.emit_host_reference_declarations!(mod, refs) - @test isempty(refs.slots) - @test !haskey(globals(mod), name) - @test occursin(r"load i64, .*@jl_float32_type", string(mod)) + @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test isdeclaration(globals(mod)[name]) + @test occursin("@$name = external global i64", string(mod)) end end From 524fb5df82b6ee33a54a3d046470a08547ecc549 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:02:15 +0200 Subject: [PATCH 16/45] Centralize host-reference slot validation --- src/relocation.jl | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/relocation.jl b/src/relocation.jl index 79b2d075..c54a9d81 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -174,10 +174,13 @@ function host_reference_slot_initializer(gv::GlobalVariable, value::UInt) error("Host reference slot '$(LLVM.name(gv))' has unsupported LLVM type $T") end -function check_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) +function foreach_host_reference_slot(f, mod::LLVM.Module, refs::HostReferences) mod_gvs = globals(mod) - for name in keys(refs.slots) + for (name, ref) in refs.slots haskey(mod_gvs, name) || error("Missing host reference slot '$name'") + gv = mod_gvs[name] + host_reference_slot_size(mod, gv, name) + f(name, gv, ref) end return end @@ -191,11 +194,7 @@ function prune_dead_host_reference_slots!(mod::LLVM.Module, refs::HostReferences end function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) - check_host_reference_slots!(mod, refs) - mod_gvs = globals(mod) - for (name, ref) in refs.slots - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) + foreach_host_reference_slot(mod, refs) do name, gv, ref value = resolve_host_reference(ref) val = host_reference_slot_initializer(gv, value) initializer!(gv, val) @@ -214,12 +213,8 @@ Emit host-reference slots as writable, null-initialized definitions. The loader definition after loading the object. This requires a per-object symbol namespace. """ function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences) - check_host_reference_slots!(mod, refs) - mod_gvs = globals(mod) slots = GlobalVariable[] - for name in keys(refs.slots) - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) + foreach_host_reference_slot(mod, refs) do _, gv, _ initializer!(gv, null(global_value_type(gv))) constant!(gv, false) linkage!(gv, LLVM.API.LLVMExternalLinkage) @@ -240,11 +235,7 @@ point at a cell containing [`resolve_host_reference`](@ref), and keep the cell a referenced Julia value alive while the code is executable. """ function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) - check_host_reference_slots!(mod, refs) - mod_gvs = globals(mod) - for name in keys(refs.slots) - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) + foreach_host_reference_slot(mod, refs) do name, gv, _ isdeclaration(gv) || error("Host reference slot '$name' must be a declaration") constant!(gv, false) linkage!(gv, LLVM.API.LLVMExternalLinkage) From a0b1112e927309a5707a228e60b04973c66d576b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:03:10 +0200 Subject: [PATCH 17/45] Remove the legacy global relocation entrypoint --- src/driver.jl | 4 ++++ src/interface.jl | 6 +++--- src/jlgen.jl | 4 ++-- src/relocation.jl | 6 ------ test/native.jl | 24 ++++++++++++++++++------ 5 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index 579a18db..b662c6ae 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -50,6 +50,10 @@ export compile Compile a `job` to one of the following formats as specified by the `target` argument: `:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. + +The returned `host_references` metadata is final only for `:asm` and `:obj`, after runtime +globals have been collected and the backend has lowered every live slot. The `:llvm` result +still contains symbolic Julia-value slots and may contain raw `jl_*` runtime references. """ function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing diff --git a/src/interface.jl b/src/interface.jl index 83c91c78..d7aacc29 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -507,9 +507,9 @@ end ## session-dependent results # -# Some compilation results embed session-specific data: `relocate_gvs!` bakes absolute -# pointers into the IR of toplevel jobs that reference `julia.constgv` globals (except -# for slots it can materialize as session-portable device constants), and any +# Some compilation results embed session-specific data: classification may materialize a +# box whose header contains a host type pointer, and eager host-reference lowering bakes +# absolute pointers into the final IR. Any # artifact a back-end derives from that IR (metallib, SPIR-V, ...) inherits them. Such # results must not survive into a package image, while remaining available for # within-session lookups during the precompilation process itself. Julia wipes its own diff --git a/src/jlgen.jl b/src/jlgen.jl index 9ecbffbf..e4d5e7c6 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -600,8 +600,8 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # Strip the initializers so the IR we hand back is session-portable # (on 1.12 `jl_emit_native_impl` bakes pointers via # `literal_static_pointer_val`; on 1.13+ Julia nulls them itself); - # `relocate_gvs!` at the toplevel link step writes the session-current - # value in. Demote each GV to an external *declaration* rather than + # host-reference lowering writes the session-current value in. Demote + # each GV to an external *declaration* rather than # giving it a null initializer: an internal global initialized to null # is fair game for optimization passes that run before relocation # (e.g. GlobalOpt in the PTX back-end's `finish_module!`), which would diff --git a/src/relocation.jl b/src/relocation.jl index c54a9d81..45716264 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -261,12 +261,6 @@ function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostRefere return end -function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - refs = classify_gvs!(mod, gv_to_value) - resolve_host_reference_slots!(mod, refs) - return !refs.embedded_pointer -end - # emit a device-resident constant replica of the box holding `obj`; returns # the constant to store in the slot, and the (gcbits-masked) header word function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(obj), diff --git a/test/native.jl b/test/native.jl index 5671f623..97e37d10 100644 --- a/test/native.jl +++ b/test/native.jl @@ -361,7 +361,8 @@ end Native.code_execution(mod.kernel, (Ptr{Int64}, Bool, Int32)) Native.code_execution(mod.egal_kernel, (Ptr{Bool}, Bool, Int32)) - # relocate_gvs! reports whether the module stayed session-portable + # Classification records whether the module stayed session-portable; eager + # lowering then resolves any remaining host-reference slots. JuliaContext() do ctx # Unlike Int128, vector-shaped tuples are 16-byte aligned on all # supported architectures and Julia versions. @@ -384,7 +385,9 @@ end gv = LLVM.GlobalVariable(m, LLVM.PointerType(LLVM.Int8Type()), name) constant!(gv, true) end - @test GPUCompiler.relocate_gvs!(m, Dict{String, Ptr{Cvoid}}()) + refs = GPUCompiler.classify_gvs!(m, Dict{String, Ptr{Cvoid}}()) + @test !refs.embedded_pointer + GPUCompiler.resolve_host_reference_slots!(m, refs) bool_ir = string(m) for name in ("jl_true", "jl_false") @test haskey(globals(m), "$(name)_box") @@ -397,26 +400,35 @@ end GC.@preserve objs begin # smalltag isbits: materialized, portable m, map = slot_module(ptrs[1]) - @test GPUCompiler.relocate_gvs!(m, map) + refs = GPUCompiler.classify_gvs!(m, map) + @test !refs.embedded_pointer + GPUCompiler.resolve_host_reference_slots!(m, refs) @test haskey(globals(m), "jl_global_0_box") dispose(m) # Float64: materialized, but the header carries a type pointer m, map = slot_module(ptrs[2]) - @test !GPUCompiler.relocate_gvs!(m, map) + refs = GPUCompiler.classify_gvs!(m, map) + @test refs.embedded_pointer + GPUCompiler.resolve_host_reference_slots!(m, refs) @test haskey(globals(m), "jl_global_0_box") dispose(m) # Symbol: baked address m, map = slot_module(ptrs[3]) - @test !GPUCompiler.relocate_gvs!(m, map) + refs = GPUCompiler.classify_gvs!(m, map) + @test !refs.embedded_pointer + @test only(values(refs.slots)).value === objs[3] + GPUCompiler.resolve_host_reference_slots!(m, refs) + @test refs.embedded_pointer @test !haskey(globals(m), "jl_global_0_box") @test occursin("inttoptr", string(m)) dispose(m) # 16-byte-aligned payloads get padded past the header word m, map = slot_module(ptrs[4]) - GPUCompiler.relocate_gvs!(m, map) + refs = GPUCompiler.classify_gvs!(m, map) + GPUCompiler.resolve_host_reference_slots!(m, refs) box = globals(m)["jl_global_0_box"] @test length(elements(LLVM.global_value_type(box))) == 3 dispose(m) From 0ea077672c84e06265802c026162bc3359ff1473 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:03:40 +0200 Subject: [PATCH 18/45] Stop threading relocation state through pipeline builders --- src/optim.jl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/optim.jl b/src/optim.jl index de35280a..5cbae38c 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -68,7 +68,7 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module, register!(pb, CleanupKernelStatePass(job)) add!(pb, NewPMModulePassManager()) do mpm - buildNewPMPipeline!(mpm, job, refs, opt_level) + buildNewPMPipeline!(mpm, job, opt_level) end run!(pb, mod, tm) end @@ -77,8 +77,7 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module, return end -function buildNewPMPipeline!(mpm, @nospecialize(job::CompilerJob), refs::HostReferences, - opt_level) +function buildNewPMPipeline!(mpm, @nospecialize(job::CompilerJob), opt_level) buildEarlySimplificationPipeline(mpm, job, opt_level) add!(mpm, AlwaysInlinerPass()) buildEarlyOptimizerPipeline(mpm, job, opt_level) @@ -92,7 +91,7 @@ function buildNewPMPipeline!(mpm, @nospecialize(job::CompilerJob), refs::HostRef add!(fpm, WarnMissedTransformationsPass()) end end - buildIntrinsicLoweringPipeline(mpm, job, refs, opt_level) + buildIntrinsicLoweringPipeline(mpm, job, opt_level) buildCleanupPipeline(mpm, job, opt_level) end @@ -318,8 +317,7 @@ function buildVectorPipeline(fpm, @nospecialize(job::CompilerJob), opt_level) add!(fpm, InstSimplifyPass()) end -function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), - refs::HostReferences, opt_level) +function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), opt_level) add!(mpm, RemoveNIPass()) # lower GC intrinsics @@ -328,7 +326,9 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), add!(fpm, GPULowerGCFramePass(job)) end if job.config.libraries - add!(mpm, GPULinkRuntimePass(job, refs)) + # The pass builder resolves this name to the registered instance, which + # captures the HostReferences object owned by optimize!. + add!(mpm, "GPULinkRuntime") add!(mpm, GPULinkLibrariesPass(job)) add!(mpm, GPUFinishRuntimeIntrinsicsPass(job)) end @@ -495,7 +495,7 @@ function (self::LinkRuntime)(mod::LLVM.Module) link_with_host_references!(mod, self.refs, runtime, runtime_refs; only_needed=true) return true end -GPULinkRuntimePass(job, refs=HostReferences()) = +GPULinkRuntimePass(job, refs::HostReferences) = NewPMModulePass("GPULinkRuntime", LinkRuntime(job, refs)) struct LinkLibraries From 1d72200968cf7166a5454965860c64bb8ddb3c3f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:42:40 +0200 Subject: [PATCH 19/45] Simplify runtime-global reference collection --- src/mcgen.jl | 7 +-- src/relocation.jl | 108 ++++++++++++++++++++-------------------------- test/native.jl | 6 +-- 3 files changed, 53 insertions(+), 68 deletions(-) diff --git a/src/mcgen.jl b/src/mcgen.jl index 6b1bdd48..fc7d6135 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -1,7 +1,6 @@ # machine code generation -# final preparations for the module to be compiled to machine code -# these passes should not be run when e.g. compiling to write to disk. +# GlobalOpt/DCE cleanup, run before slot collection and again after lowering. function run_cleanup_pipeline!(@nospecialize(job::CompilerJob), mod::LLVM.Module) @dispose pb=NewPMPassBuilder() begin add!(pb, RecomputeGlobalsAAPass()) @@ -13,13 +12,15 @@ function run_cleanup_pipeline!(@nospecialize(job::CompilerJob), mod::LLVM.Module return end +# Final preparations for the module to be compiled to machine code. These passes should not +# be run when e.g. compiling to write to disk. function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, refs::HostReferences=HostReferences()) # Clean up first so only live references get slots and get lowered. run_cleanup_pipeline!(job, mod) prune_dead_host_reference_slots!(mod, refs) - collect_runtime_global_references!(job, mod, refs) + collect_runtime_global_references!(mod, refs) lower_host_references!(job, mod, refs) # Fold constants exposed by eager lowering, and discard slots made dead by either diff --git a/src/relocation.jl b/src/relocation.jl index 45716264..49384058 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -121,10 +121,6 @@ function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) else host_reference_slot_size(mod, gv, name) slot_name = safe_name(name) * "_" * string(objectid(obj); base=16) - if slot_name != name && - (haskey(globals(mod), slot_name) || haskey(functions(mod), slot_name)) - error("Host reference slot name '$slot_name' is already in use") - end LLVM.name!(gv, slot_name) LLVM.name(gv) == slot_name || error("Host reference slot name '$slot_name' is already in use") @@ -310,75 +306,67 @@ end # since those references are ephemeral, we can't eagerly resolve and emit them in the IR, # but at the same time the GPU can't resolve them at run-time. # -# this pass performs that resolution at link time. -struct CollectRuntimeGlobalReferences - job::CompilerJob - refs::HostReferences +# this collection performs that resolution at object-emission time. +function is_runtime_global_candidate(value, refs::HostReferences) + name = LLVM.name(value) + value isa LLVM.GlobalVariable && haskey(refs.slots, name) && return false + isdeclaration(value) || return false + value isa LLVM.Function && LLVM.isintrinsic(value) && return false + return startswith(name, "jl_") end -function (self::CollectRuntimeGlobalReferences)(mod::LLVM.Module) + +function collect_runtime_global_references!(mod::LLVM.Module, refs::HostReferences) changed = false for f in [collect(functions(mod)); collect(globals(mod))] + is_runtime_global_candidate(f, refs) || continue fn = LLVM.name(f) - # Julia value globals are already represented by an identity in `refs`; they may - # use a `jl_*` spelling but are not exported libjulia runtime globals. - f isa LLVM.GlobalVariable && haskey(self.refs.slots, fn) && continue - if isdeclaration(f) && (!(f isa LLVM.Function) || !LLVM.isintrinsic(f)) && - startswith(fn, "jl_") - slot = nothing - function runtime_global_slot() - if slot === nothing - name = "gpu_" * fn - (haskey(globals(mod), name) || haskey(functions(mod), name)) && - error("Julia runtime global slot name '$name' is already in use") - slot = GlobalVariable(mod, host_reference_word_type(), name) - LLVM.name(slot) == name || - error("Julia runtime global slot name '$name' is already in use") - haskey(self.refs.slots, name) && - error("Duplicate Julia runtime global slot '$name'") - self.refs.slots[name] = CGlobalRef(Symbol(fn)) - end - slot + slot = nothing + function runtime_global_slot() + if slot === nothing + name = "gpu_" * fn + slot = GlobalVariable(mod, host_reference_word_type(), name) + LLVM.name(slot) == name || + error("Julia runtime global slot name '$name' is already in use") + refs.slots[name] = CGlobalRef(Symbol(fn)) end + slot + end - function replace_bindings!(value) - changed = false - for use in collect(uses(value)) - val = user(use) - if isa(val, LLVM.ConstantExpr) - # recurse - changed |= replace_bindings!(val) - elseif isa(val, LLVM.LoadInst) - T = value_type(val) - if !(T isa LLVM.PointerType || - (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) - error("Unsupported Julia runtime global '$fn' load of LLVM type $T") - end - @dispose builder=IRBuilder() begin - position!(builder, val) - replacement = load!(builder, host_reference_word_type(), - runtime_global_slot()) - if T isa LLVM.PointerType - replacement = inttoptr!(builder, replacement, T) - end - replace_uses!(val, replacement) + function replace_bindings!(value) + changed = false + for use in collect(uses(value)) + val = user(use) + if isa(val, LLVM.ConstantExpr) + # recurse + changed |= replace_bindings!(val) + elseif isa(val, LLVM.LoadInst) + T = value_type(val) + if !(T isa LLVM.PointerType || + (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) + error("Unsupported Julia runtime global '$fn' load of LLVM type $T") + end + @dispose builder=IRBuilder() begin + position!(builder, val) + replacement = load!(builder, host_reference_word_type(), + runtime_global_slot()) + if T isa LLVM.PointerType + replacement = inttoptr!(builder, replacement, T) end - erase!(val) - changed = true + replace_uses!(val, replacement) end + erase!(val) + changed = true end - changed end - - changed |= replace_bindings!(f) + changed end + + changed |= replace_bindings!(f) end return changed end -collect_runtime_global_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences) = - CollectRuntimeGlobalReferences(job, refs)(mod) function has_unresolved_runtime_global_loads(mod::LLVM.Module, refs::HostReferences) function has_load(value) @@ -391,11 +379,7 @@ function has_unresolved_runtime_global_loads(mod::LLVM.Module, refs::HostReferen end for value in [collect(functions(mod)); collect(globals(mod))] - name = LLVM.name(value) - value isa LLVM.GlobalVariable && haskey(refs.slots, name) && continue - isdeclaration(value) || continue - value isa LLVM.Function && LLVM.isintrinsic(value) && continue - startswith(name, "jl_") || continue + is_runtime_global_candidate(value, refs) || continue has_load(value) && return true end return false diff --git a/test/native.jl b/test/native.jl index 97e37d10..54f0aeb9 100644 --- a/test/native.jl +++ b/test/native.jl @@ -820,7 +820,7 @@ end ret $word_ptr %value }""") refs = GPUCompiler.HostReferences() - @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) + @test GPUCompiler.collect_runtime_global_references!(mod, refs) name = only(keys(refs.slots)) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) @@ -835,7 +835,7 @@ end ret i64 %value }""") refs = GPUCompiler.HostReferences() - @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) + @test GPUCompiler.collect_runtime_global_references!(mod, refs) name = only(keys(refs.slots)) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) @@ -852,7 +852,7 @@ end ret $word_ptr %value }""") refs = GPUCompiler.HostReferences() - @test GPUCompiler.collect_runtime_global_references!(job, mod, refs) + @test GPUCompiler.collect_runtime_global_references!(mod, refs) name = only(keys(refs.slots)) GPUCompiler.emit_host_reference_declarations!(mod, refs) @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) From 275618586cf4a37775936731a24f01cc8b2b7b20 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:43:07 +0200 Subject: [PATCH 20/45] Mark LLVM results after runtime linking --- src/driver.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index b662c6ae..473de370 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -337,8 +337,6 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) - host_references.embedded_pointer && mark_session_dependent!(job) - if job.config.optimize @tracepoint "optimization" begin optimize!(job, ir, host_references; job.config.opt_level) @@ -404,6 +402,9 @@ const __llvm_initialized = Ref(false) end end + # Optimization may link runtime functions carrying embedded host pointers. + job.config.toplevel && host_references.embedded_pointer && mark_session_dependent!(job) + if job.config.toplevel && job.config.validate @tracepoint "validation" begin check_ir(job, ir, host_references) From 540ebf2f67b85582a76cc25a9ea2e6361fd7ab06 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 19:43:27 +0200 Subject: [PATCH 21/45] Name both host-reference producers consistently --- src/irgen.jl | 2 +- src/relocation.jl | 6 +++++- test/native.jl | 11 ++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/irgen.jl b/src/irgen.jl index c5dc218f..702c0a3e 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -2,7 +2,7 @@ function irgen(@nospecialize(job::CompilerJob)) mod, compiled, gv_to_value = @tracepoint "emission" compile_method_instance(job) - host_references = classify_gvs!(mod, gv_to_value) + host_references = collect_julia_value_references!(mod, gv_to_value) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else diff --git a/src/relocation.jl b/src/relocation.jl index 49384058..3bb747e3 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -8,6 +8,9 @@ # C symbol for CGlobalRef slots. Same name therefore means the same HostReference, so linking # can merge declarations and metadata without renaming either one. # +# The two producers are `collect_julia_value_references!` during IR generation and +# `collect_runtime_global_references!` immediately before backend lowering. +# # This mirrors Julia's own mechanisms: codegen's identity-keyed global_targets slots, the # sysimage jl_gvars table patched by jl_update_all_gvars, and JIT absoluteSymbols definitions. @@ -97,7 +100,8 @@ function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Modul end -function classify_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) +function collect_julia_value_references!(mod::LLVM.Module, + gv_to_value::Dict{String, Ptr{Cvoid}}) refs = HostReferences() mod_gvs = globals(mod) for (name, init) in gv_to_value diff --git a/test/native.jl b/test/native.jl index 54f0aeb9..9d834d7d 100644 --- a/test/native.jl +++ b/test/native.jl @@ -385,7 +385,8 @@ end gv = LLVM.GlobalVariable(m, LLVM.PointerType(LLVM.Int8Type()), name) constant!(gv, true) end - refs = GPUCompiler.classify_gvs!(m, Dict{String, Ptr{Cvoid}}()) + refs = GPUCompiler.collect_julia_value_references!( + m, Dict{String, Ptr{Cvoid}}()) @test !refs.embedded_pointer GPUCompiler.resolve_host_reference_slots!(m, refs) bool_ir = string(m) @@ -400,7 +401,7 @@ end GC.@preserve objs begin # smalltag isbits: materialized, portable m, map = slot_module(ptrs[1]) - refs = GPUCompiler.classify_gvs!(m, map) + refs = GPUCompiler.collect_julia_value_references!(m, map) @test !refs.embedded_pointer GPUCompiler.resolve_host_reference_slots!(m, refs) @test haskey(globals(m), "jl_global_0_box") @@ -408,7 +409,7 @@ end # Float64: materialized, but the header carries a type pointer m, map = slot_module(ptrs[2]) - refs = GPUCompiler.classify_gvs!(m, map) + refs = GPUCompiler.collect_julia_value_references!(m, map) @test refs.embedded_pointer GPUCompiler.resolve_host_reference_slots!(m, refs) @test haskey(globals(m), "jl_global_0_box") @@ -416,7 +417,7 @@ end # Symbol: baked address m, map = slot_module(ptrs[3]) - refs = GPUCompiler.classify_gvs!(m, map) + refs = GPUCompiler.collect_julia_value_references!(m, map) @test !refs.embedded_pointer @test only(values(refs.slots)).value === objs[3] GPUCompiler.resolve_host_reference_slots!(m, refs) @@ -427,7 +428,7 @@ end # 16-byte-aligned payloads get padded past the header word m, map = slot_module(ptrs[4]) - refs = GPUCompiler.classify_gvs!(m, map) + refs = GPUCompiler.collect_julia_value_references!(m, map) GPUCompiler.resolve_host_reference_slots!(m, refs) box = globals(m)["jl_global_0_box"] @test length(elements(LLVM.global_value_type(box))) == 3 From 28521102bd24bf12bd50d9b8c0db3725c0d8b5b9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 20:16:58 +0200 Subject: [PATCH 22/45] Represent all host pointers as relocations --- src/deprecated.jl | 11 +-- src/driver.jl | 4 -- src/interface.jl | 51 ++----------- src/mcgen.jl | 4 +- src/relocation.jl | 148 ++++++++++++++++++++++++++++---------- src/rtlib.jl | 12 ++-- src/utils.jl | 7 ++ test/native.jl | 63 ++++++++-------- test/native/precompile.jl | 20 +++--- 9 files changed, 177 insertions(+), 143 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index d3c649eb..7f41cfb4 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -272,16 +272,11 @@ function cached_results(::Type{V}, job::CompilerJob) where {V} end -## 1.10 session-dependent results -# -# Nothing to wipe: `legacy_job_results` never persists meaningfully across sessions (its -# CodeInstance keys are session-specific, and our own image's entries are cleared in -# `__init__`; entries written by a downstream package's workload don't make it into that -# package's image at all, cf. cross-image mutation loss). -mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing - end # !HAS_INTEGRATED_CACHE +# Compatibility shim for backends using the former dynamic cache-wiping API. +mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing + ## Legacy `cached_compilation` (1.10+) diff --git a/src/driver.jl b/src/driver.jl index 473de370..9a308baa 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -402,9 +402,6 @@ const __llvm_initialized = Ref(false) end end - # Optimization may link runtime functions carrying embedded host pointers. - job.config.toplevel && host_references.embedded_pointer && mark_session_dependent!(job) - if job.config.toplevel && job.config.validate @tracepoint "validation" begin check_ir(job, ir, host_references) @@ -427,7 +424,6 @@ end @tracepoint "LLVM back-end" begin @tracepoint "preparation" prepare_execution!(job, ir, refs) - refs.embedded_pointer && mark_session_dependent!(job) code = @tracepoint "machine-code generation" mcgen(job, ir, format) end diff --git a/src/interface.jl b/src/interface.jl index d7aacc29..fb99cda6 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -430,6 +430,12 @@ the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock function cached_results end # Host-reference types and the backend lowering hook live in relocation.jl. +# +# Job results attached to CodeInstances are serialized into package images. Store only +# session-portable data there. Generated code is portable iff `supports_relocatable_ir()` +# and the backend preserves host references with definition- or declaration-based lowering, +# then patches them at load time. Backends using eager lowering must keep generated code in a +# session-local cache or recompile it. @static if HAS_INTEGRATED_CACHE """ @@ -505,52 +511,9 @@ function cached_results(::Type{V}, job::CompilerJob) where {V} return job_results(V, ci, job.config) end -## session-dependent results -# -# Some compilation results embed session-specific data: classification may materialize a -# box whose header contains a host type pointer, and eager host-reference lowering bakes -# absolute pointers into the final IR. Any -# artifact a back-end derives from that IR (metallib, SPIR-V, ...) inherits them. Such -# results must not survive into a package image, while remaining available for -# within-session lookups during the precompilation process itself. Julia wipes its own -# session-dependent CodeInstance state during serialization (staticdata.c); we -# approximate that with an `atexit` hook, which the runtime invokes *before* -# `jl_write_compiler_output`: right before the image is written, the entries of jobs -# marked session-dependent are deleted from their `JobResults` container, so a later -# session simply recompiles them. - -const session_dependent_jobs = Vector{CompilerJob}() -const session_dependent_lock = ReentrantLock() - -function mark_session_dependent!(@nospecialize(job::CompilerJob)) - ccall(:jl_generating_output, Cint, ()) == 1 || return - Base.@lock session_dependent_lock begin - if isempty(session_dependent_jobs) - atexit(wipe_session_dependent_results) - end - push!(session_dependent_jobs, job) - end - return -end - -function wipe_session_dependent_results() - Base.@lock session_dependent_lock begin - for job in session_dependent_jobs - ci = job_code_instance(job) - ci === nothing && continue - jr = CompilerCaching.results(JobResults, ci) - Base.@lock cached_results_lock begin - filter!(entry -> entry.config !== job.config, jr.entries) - end - end - empty!(session_dependent_jobs) - end - return -end - end # HAS_INTEGRATED_CACHE -@public GPUCompilerCacheToken, cache_owner, cached_results +@public GPUCompilerCacheToken, cache_owner, cached_results, supports_relocatable_ir @public JuliaValueRef, CGlobalRef, HostReference, HostReferences @public lower_host_references!, emit_host_reference_definitions! @public emit_host_reference_declarations! diff --git a/src/mcgen.jl b/src/mcgen.jl index fc7d6135..9b754182 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -18,7 +18,7 @@ function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, refs::HostReferences=HostReferences()) # Clean up first so only live references get slots and get lowered. run_cleanup_pipeline!(job, mod) - prune_dead_host_reference_slots!(mod, refs) + prune_dead_host_references!(mod, refs) collect_runtime_global_references!(mod, refs) lower_host_references!(job, mod, refs) @@ -26,7 +26,7 @@ function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, # Fold constants exposed by eager lowering, and discard slots made dead by either # lowering strategy. run_cleanup_pipeline!(job, mod) - prune_dead_host_reference_slots!(mod, refs) + prune_dead_host_references!(mod, refs) has_unresolved_runtime_global_loads(mod, refs) && error("Unresolved Julia runtime global load after host-reference lowering") diff --git a/src/relocation.jl b/src/relocation.jl index 3bb747e3..42c71b6a 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -2,15 +2,16 @@ # # produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) # -# Julia value globals and libjulia runtime globals are represented by symbolic slots until -# the backend lowers them. A slot's LLVM name is globally unique and assigned exactly once: -# the readable Julia name plus object identity for JuliaValueRef slots, and `gpu_` plus the -# C symbol for CGlobalRef slots. Same name therefore means the same HostReference, so linking -# can merge declarations and metadata without renaming either one. +# Julia values produce two relocation kinds: word-sized declaration slots and patches at a +# byte offset inside a defined global. Runtime globals produce slots. Names are globally +# unique and assigned once, so linking can merge IR and metadata without renaming. # # The two producers are `collect_julia_value_references!` during IR generation and # `collect_runtime_global_references!` immediately before backend lowering. # +# Generated code is portable only when `supports_relocatable_ir()` and the backend preserves +# these relocations for its loader. The default eager lowering bakes current-session values. +# # This mirrors Julia's own mechanisms: codegen's identity-keyed global_targets slots, the # sysimage jl_gvars table patched by jl_update_all_gvars, and JIT absoluteSymbols definitions. @@ -51,19 +52,19 @@ A serializable source for a host-derived word: either a [`JuliaValueRef`](@ref) const HostReference = Union{JuliaValueRef,CGlobalRef} """ - HostReferences(slots, embedded_pointer) + HostReferences(slots, patches) -Host-reference metadata accompanying a module. `slots` maps each object-code symbol to the -source of the word that must be written there. `embedded_pointer` conservatively records that -an unavoidable session pointer has been embedded, so the artifact cannot be serialized into a -package image even if `slots` is empty. +Host-reference metadata accompanying a module. `slots` maps word-sized declaration symbols to +the word resolved by a loader. `patches` maps a global name and byte offset to a word patched +after loading. """ -mutable struct HostReferences +struct HostReferences slots::Dict{String,HostReference} - embedded_pointer::Bool + patches::Dict{Tuple{String,Int},HostReference} end -HostReferences() = HostReferences(Dict{String,HostReference}(), false) +HostReferences() = HostReferences(Dict{String,HostReference}(), + Dict{Tuple{String,Int},HostReference}()) same_host_reference(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value same_host_reference(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol @@ -93,10 +94,15 @@ Backend hook for lowering live host references before object emission. The default implementation resolves them in the current Julia process, making the result session-dependent. Loaders may instead emit module-owned definitions that are patched after loading, or external declarations that are defined before loading. + +Backends using eager lowering must not persist generated code in `cached_results`. Generated +code is session-portable only when [`supports_relocatable_ir`](@ref) and the backend preserves +all slots and patches with definition- or declaration-based lowering, then applies them at +load time. """ function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, refs::HostReferences) - resolve_host_reference_slots!(mod, refs) + resolve_host_references!(mod, refs) end @@ -109,7 +115,7 @@ function collect_julia_value_references!(mod::LLVM.Module, gv = mod_gvs[name] cur = initializer(gv) if !(cur === nothing || LLVM.isnull(cur)) - refs.embedded_pointer = true + @assert !supports_relocatable_ir() continue end @@ -118,10 +124,9 @@ function collect_julia_value_references!(mod::LLVM.Module, init == C_NULL && error("Missing Julia object for global '$name'") obj = Base.unsafe_pointer_to_objref(init) if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) - val, hdr = materialize_box!(mod, gv, obj, init) + val = materialize_box!(mod, refs, gv, obj, init) initializer!(gv, val) linkage!(gv, LLVM.API.LLVMPrivateLinkage) - refs.embedded_pointer |= hdr >= UInt(64 << 4) else host_reference_slot_size(mod, gv, name) slot_name = safe_name(name) * "_" * string(objectid(obj); base=16) @@ -138,19 +143,15 @@ function collect_julia_value_references!(mod::LLVM.Module, gv = mod_gvs[name] cur = initializer(gv) if !(cur === nothing || LLVM.isnull(cur)) - # Existing definitions may contain session-specific addresses. - refs.embedded_pointer = true + @assert !supports_relocatable_ir() continue end init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) - val, hdr = materialize_box!(mod, gv, obj, init) + val = materialize_box!(mod, refs, gv, obj, init) initializer!(gv, val) constant!(gv, true) linkage!(gv, LLVM.API.LLVMPrivateLinkage) - - # Stay conservative if Bool stops using a smalltag. - refs.embedded_pointer |= hdr >= UInt(64 << 4) # jl_max_tags << 4 end return refs end @@ -185,24 +186,62 @@ function foreach_host_reference_slot(f, mod::LLVM.Module, refs::HostReferences) return end -function prune_dead_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) +function foreach_host_reference_patch(f, mod::LLVM.Module, refs::HostReferences) + mod_gvs = globals(mod) + for ((name, offset), ref) in refs.patches + haskey(mod_gvs, name) || error("Missing host reference patch global '$name'") + gv = mod_gvs[name] + init = initializer(gv) + init === nothing && error("Host reference patch global '$name' has no initializer") + T = value_type(init) + T isa LLVM.StructType || + error("Host reference patch global '$name' has non-struct initializer $T") + size = abi_size(datalayout(mod), T) + 0 <= offset && offset + sizeof(UInt) <= size || + error("Host reference patch '$name+$offset' is outside its $size-byte global") + f(name, offset, gv, ref) + end + return +end + +function prune_dead_host_references!(mod::LLVM.Module, refs::HostReferences) mod_gvs = globals(mod) for name in collect(keys(refs.slots)) haskey(mod_gvs, name) || delete!(refs.slots, name) end + for ((name, offset), _) in collect(refs.patches) + if !haskey(mod_gvs, name) + delete!(refs.patches, (name, offset)) + elseif isempty(uses(mod_gvs[name])) + delete!(refs.patches, (name, offset)) + erase!(mod_gvs[name]) + end + end return end -function resolve_host_reference_slots!(mod::LLVM.Module, refs::HostReferences) +function resolve_host_references!(mod::LLVM.Module, refs::HostReferences) foreach_host_reference_slot(mod, refs) do name, gv, ref value = resolve_host_reference(ref) val = host_reference_slot_initializer(gv, value) initializer!(gv, val) linkage!(gv, LLVM.API.LLVMPrivateLinkage) constant!(gv, true) - refs.embedded_pointer = true + end + foreach_host_reference_patch(mod, refs) do name, offset, gv, ref + init = initializer(gv) + T = value_type(init)::LLVM.StructType + idx = Int(element_at(datalayout(mod), T, offset)) + 1 + fields = LLVM.Constant[operands(init)...] + fields[idx] = ConstantInt(value_type(fields[idx]), resolve_host_reference(ref)) + initializer!(gv, ConstantStruct(T, fields)) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + extinit!(gv, false) + constant!(gv, true) + unnamed_addr!(gv, true) end empty!(refs.slots) + empty!(refs.patches) return end @@ -210,7 +249,8 @@ end emit_host_reference_definitions!(mod, refs) Emit host-reference slots as writable, null-initialized definitions. The loader must patch each -definition after loading the object. This requires a per-object symbol namespace. +definition and interior relocation after loading the object. This requires a per-object symbol +namespace. """ function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences) slots = GlobalVariable[] @@ -221,6 +261,9 @@ function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences extinit!(gv, true) push!(slots, gv) end + foreach_host_reference_patch(mod, refs) do _, _, gv, _ + push!(slots, gv) + end isempty(slots) || set_used!(mod, slots...) return end @@ -232,15 +275,21 @@ Prepare host references for a loader that defines symbols before loading an obje Every slot remains an external word-sized declaration. The loader must define each symbol to point at a cell containing [`resolve_host_reference`](@ref), and keep the cell and any -referenced Julia value alive while the code is executable. +referenced Julia value alive while the code is executable. Interior patches cannot be defined +before load and must always be written afterward. """ function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) + used = GlobalVariable[] foreach_host_reference_slot(mod, refs) do name, gv, _ isdeclaration(gv) || error("Host reference slot '$name' must be a declaration") constant!(gv, false) linkage!(gv, LLVM.API.LLVMExternalLinkage) extinit!(gv, false) end + foreach_host_reference_patch(mod, refs) do _, _, gv, _ + push!(used, gv) + end + isempty(used) || set_used!(mod, used...) return end @@ -257,14 +306,21 @@ function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostRefere error("Host reference slot '$name' refers to conflicting values") dest_refs.slots[name] = ref end - dest_refs.embedded_pointer |= src_refs.embedded_pointer + for ((name, offset), ref) in src_refs.patches + haskey(globals(dest_mod), name) || continue + key = (name, offset) + existing = get(dest_refs.patches, key, nothing) + existing === nothing || same_host_reference(existing, ref) || + error("Host reference patch '$name+$offset' refers to conflicting values") + dest_refs.patches[key] = ref + end return end # emit a device-resident constant replica of the box holding `obj`; returns # the constant to store in the slot, and the (gcbits-masked) header word -function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(obj), - init::Ptr{Cvoid}) +function materialize_box!(mod::LLVM.Module, refs::HostReferences, gv::GlobalVariable, + @nospecialize(obj), init::Ptr{Cvoid}) @assert isbitstype(typeof(obj)) && sizeof(obj) > 0 W = sizeof(Int) @@ -278,28 +334,46 @@ function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(ob T_word = LLVM.IntType(8W) T_byte = LLVM.Int8Type() - fields = LLVM.Constant[ConstantInt(T_word, hdr), ConstantDataArray(T_byte, bytes)] + patch_header = hdr >= UInt(64 << 4) # jl_max_tags << 4 + fields = LLVM.Constant[ConstantInt(T_word, patch_header ? 0 : hdr), + ConstantDataArray(T_byte, bytes)] + header_idx = 0 payload_idx = 1 if Base.datatype_alignment(typeof(obj)) > W # pad so the payload lands at a 16-byte offset (JL_HEAP_ALIGNMENT max) pushfirst!(fields, ConstantDataArray(T_byte, zeros(UInt8, 16 - W))) + header_idx = 1 payload_idx = 2 end boxinit = ConstantStruct(fields) boxty = value_type(boxinit) - box = GlobalVariable(mod, boxty, safe_name(LLVM.name(gv)) * "_box") + box_name = if patch_header + safe_name(LLVM.name(gv)) * "_" * string(objectid(obj); base=16) * "_box" + else + safe_name(LLVM.name(gv)) * "_box" + end + box = GlobalVariable(mod, boxty, box_name) + LLVM.name(box) == box_name || error("Host reference patch global '$box_name' is already in use") initializer!(box, boxinit) - constant!(box, true) - linkage!(box, LLVM.API.LLVMPrivateLinkage) alignment!(box, 16) - unnamed_addr!(box, true) + if patch_header + constant!(box, false) + linkage!(box, LLVM.API.LLVMExternalLinkage) + extinit!(box, true) + offset = Int(offsetof(datalayout(mod), boxty, header_idx)) + refs.patches[(box_name, offset)] = JuliaValueRef(typeof(obj)) + else + constant!(box, true) + linkage!(box, LLVM.API.LLVMPrivateLinkage) + unnamed_addr!(box, true) + end idx(i) = ConstantInt(LLVM.Int32Type(), i) payload = const_gep(boxty, box, LLVM.Constant[idx(0), idx(payload_idx)]) slotty = global_value_type(gv) val = value_type(payload) == slotty ? payload : const_addrspacecast(payload, slotty) - return val, hdr + return val end diff --git a/src/rtlib.jl b/src/rtlib.jl index d5baa043..912d6aa1 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -100,9 +100,7 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - prune_dead_host_reference_slots!(new_mod, meta.host_references) - - meta.host_references.embedded_pointer && mark_session_dependent!(rt_job) + prune_dead_host_references!(new_mod, meta.host_references) # rename to the final `gpu_*` name on the per-function module, so the cached bitcode # is immediately link-ready (no per-session rename pass on a cache hit). @@ -117,9 +115,11 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, io = IOBuffer() write(io, new_mod) ci === nothing && (ci = runtime_code_instance(rt_job)) - res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) - res.bitcode = take!(io) - res.host_references = meta.host_references + if supports_relocatable_ir() + res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) + res.bitcode = take!(io) + res.host_references = meta.host_references + end link_with_host_references!(mod, refs, new_mod, meta.host_references) return ci::CodeInstance diff --git a/src/utils.jl b/src/utils.jl index db2fc7d7..83be6806 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -418,3 +418,10 @@ end return inits end end + +"""Whether Julia exposes enough global-variable metadata to emit relocatable IR.""" +supports_relocatable_ir() = @static if VERSION >= v"1.13.0-DEV.623" + true +else + HAS_LLVM_GVS_GLOBALS +end diff --git a/test/native.jl b/test/native.jl index 9d834d7d..a2bbc841 100644 --- a/test/native.jl +++ b/test/native.jl @@ -217,21 +217,8 @@ end @test new_res !== res @test new_res.asm === nothing - @static if GPUCompiler.HAS_INTEGRATED_CACHE - # session-dependent results (e.g. artifacts with relocated GVs) are wiped - # before image serialization; emulate the atexit-driven wipe directly - new_res.asm = "session-dependent" - other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") - other_res = GPUCompiler.cached_results(mod.Results, other_job) - push!(GPUCompiler.session_dependent_jobs, new_job) - GPUCompiler.wipe_session_dependent_results() - @test isempty(GPUCompiler.session_dependent_jobs) - wiped_res = GPUCompiler.cached_results(mod.Results, new_job) - @test wiped_res !== new_res - @test wiped_res.asm === nothing - # ... without affecting other configs on the same CI - @test GPUCompiler.cached_results(mod.Results, other_job) === other_res - end + @test !isdefined(GPUCompiler, :session_dependent_jobs) + @test !isdefined(GPUCompiler, :wipe_session_dependent_results) end @testset "runtime cache invalidation" begin @@ -387,8 +374,8 @@ end end refs = GPUCompiler.collect_julia_value_references!( m, Dict{String, Ptr{Cvoid}}()) - @test !refs.embedded_pointer - GPUCompiler.resolve_host_reference_slots!(m, refs) + @test isempty(refs.patches) + GPUCompiler.resolve_host_references!(m, refs) bool_ir = string(m) for name in ("jl_true", "jl_false") @test haskey(globals(m), "$(name)_box") @@ -402,26 +389,38 @@ end # smalltag isbits: materialized, portable m, map = slot_module(ptrs[1]) refs = GPUCompiler.collect_julia_value_references!(m, map) - @test !refs.embedded_pointer - GPUCompiler.resolve_host_reference_slots!(m, refs) + @test isempty(refs.patches) + GPUCompiler.resolve_host_references!(m, refs) @test haskey(globals(m), "jl_global_0_box") dispose(m) - # Float64: materialized, but the header carries a type pointer + # Float64: the non-smalltag header is an interior relocation. m, map = slot_module(ptrs[2]) refs = GPUCompiler.collect_julia_value_references!(m, map) - @test refs.embedded_pointer - GPUCompiler.resolve_host_reference_slots!(m, refs) - @test haskey(globals(m), "jl_global_0_box") + @test length(refs.patches) == 1 + (box_name, offset), ref = only(refs.patches) + @test offset == 0 + @test ref.value === Float64 + box = globals(m)[box_name] + @test isextinit(box) + @test linkage(box) == LLVM.API.LLVMExternalLinkage + header_idx = Int(element_at(datalayout(m), global_value_type(box), offset)) + 1 + @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == 0 + GPUCompiler.resolve_host_references!(m, refs) + @test isempty(refs.patches) + @test !isextinit(box) + @test isconstant(box) + @test linkage(box) == LLVM.API.LLVMPrivateLinkage + @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == + GPUCompiler.resolve_host_reference(ref) dispose(m) # Symbol: baked address m, map = slot_module(ptrs[3]) refs = GPUCompiler.collect_julia_value_references!(m, map) - @test !refs.embedded_pointer @test only(values(refs.slots)).value === objs[3] - GPUCompiler.resolve_host_reference_slots!(m, refs) - @test refs.embedded_pointer + GPUCompiler.resolve_host_references!(m, refs) + @test isempty(refs.slots) @test !haskey(globals(m), "jl_global_0_box") @test occursin("inttoptr", string(m)) dispose(m) @@ -429,8 +428,10 @@ end # 16-byte-aligned payloads get padded past the header word m, map = slot_module(ptrs[4]) refs = GPUCompiler.collect_julia_value_references!(m, map) - GPUCompiler.resolve_host_reference_slots!(m, refs) - box = globals(m)["jl_global_0_box"] + (box_name, offset), _ = only(refs.patches) + @test offset == 8 + GPUCompiler.resolve_host_references!(m, refs) + box = globals(m)[box_name] @test length(elements(LLVM.global_value_type(box))) == 3 dispose(m) end @@ -759,7 +760,6 @@ end refs = meta.host_references @test !isempty(refs.slots) @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(refs.slots)) - @test !refs.embedded_pointer @test all(name -> isdeclaration(globals(meta.ir)[name]), keys(refs.slots)) bytes = Vector{UInt8}(codeunits(obj)) @@ -877,7 +877,8 @@ end end refs(name, value) = - GPUCompiler.HostReferences(Dict(name => GPUCompiler.JuliaValueRef(value)), false) + GPUCompiler.HostReferences(Dict(name => GPUCompiler.JuliaValueRef(value)), + Dict{Tuple{String,Int},GPUCompiler.HostReference}()) # Equal Julia identities deliberately share a single slot. dest = slot_module("slot", "first") @@ -916,7 +917,7 @@ end src_refs = GPUCompiler.HostReferences(Dict( "used" => GPUCompiler.JuliaValueRef(:used), "unused" => GPUCompiler.JuliaValueRef(:unused), - ), false) + ), Dict{Tuple{String,Int},GPUCompiler.HostReference}()) dest_refs = GPUCompiler.HostReferences() GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs; only_needed=true) diff --git a/test/native/precompile.jl b/test/native/precompile.jl index eb9abc4f..cd257142 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -38,21 +38,19 @@ precompile_test_harness("Inference caching") do load_path end portable_kernel(x) = x + 1 - session_kernel(x) = x + 2 + cached_kernel(x) = x + 2 - # Attach representative back-end artifacts while the package image is built. The - # portable entry should survive serialization; the session-dependent one should be - # removed by GPUCompiler's pre-output atexit hook. + # Job results are serialized as-is. Backends are responsible for storing only + # artifacts allowed by the static relocatability contract. let job, _ = NativeCompiler.Native.create_job(portable_kernel, (Int,)) precompile(job) NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "portable" end let - job, _ = NativeCompiler.Native.create_job(session_kernel, (Int,)) + job, _ = NativeCompiler.Native.create_job(cached_kernel, (Int,)) precompile(job) - NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "session" - NativeCompiler.GPUCompiler.mark_session_dependent!(job) + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "cached" end let @@ -116,10 +114,10 @@ precompile_test_harness("Inference caching") do load_path @test portable_res !== nothing @test portable_res.artifact == "portable" - session_job, _ = NativeCompiler.Native.create_job(NativeBackend.session_kernel, (Int,)) - session_res = GPUCompiler.cached_results(NativeBackend.Results, session_job) - @test session_res !== nothing - @test session_res.artifact === nothing + cached_job, _ = NativeCompiler.Native.create_job(NativeBackend.cached_kernel, (Int,)) + cached_res = GPUCompiler.cached_results(NativeBackend.Results, cached_job) + @test cached_res !== nothing + @test cached_res.artifact == "cached" # Check that kernel survived kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel), Tuple{Vector{Int}, Int}) From bc23597d476e53f98926f5efa032136aed3390c2 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 20:22:21 +0200 Subject: [PATCH 23/45] Resolve whole and interior relocations for loaders --- src/interface.jl | 2 +- src/relocation.jl | 39 ++++++++++++++++++++++ test/helpers/native.jl | 15 +++++---- test/native.jl | 74 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+), 7 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index fb99cda6..a042f097 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -517,7 +517,7 @@ end # HAS_INTEGRATED_CACHE @public JuliaValueRef, CGlobalRef, HostReference, HostReferences @public lower_host_references!, emit_host_reference_definitions! @public emit_host_reference_declarations! -@public resolve_host_reference +@public resolve_host_reference, resolved_relocations # the method table to use # diff --git a/src/relocation.jl b/src/relocation.jl index 42c71b6a..6c26f9fa 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -86,6 +86,27 @@ function resolve_host_reference(ref::CGlobalRef) return unsafe_load(address) end +""" + resolved_relocations(refs) + +Resolve relocation metadata for a loader. Returns resolved `slots`, resolved `patches`, and +Julia `roots` that must stay alive while the loaded code can access them. +""" +function resolved_relocations(refs::HostReferences) + slots = Pair{String,UInt}[] + patches = Pair{Tuple{String,Int},UInt}[] + roots = Any[] + for (name, ref) in refs.slots + push!(slots, name => resolve_host_reference(ref)) + ref isa JuliaValueRef && push!(roots, ref.value) + end + for (key, ref) in refs.patches + push!(patches, key => resolve_host_reference(ref)) + ref isa JuliaValueRef && push!(roots, ref.value) + end + return (; slots, patches, roots) +end + """ lower_host_references!(job, mod, refs) @@ -296,6 +317,24 @@ end function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, src_mod::LLVM.Module, src_refs::HostReferences; only_needed=false) + # Patch globals are definitions, unlike slots. Make an identical source definition a + # declaration so LLVM can resolve it to the destination definition while linking. + for (key, ref) in src_refs.patches + existing = get(dest_refs.patches, key, nothing) + existing === nothing && continue + name, offset = key + same_host_reference(existing, ref) || + error("Host reference patch '$name+$offset' refers to conflicting values") + haskey(globals(dest_mod), name) || + error("Missing destination host reference patch global '$name'") + haskey(globals(src_mod), name) || + error("Missing source host reference patch global '$name'") + gv = globals(src_mod)[name] + initializer!(gv, nothing) + extinit!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + end + link!(dest_mod, src_mod; only_needed) for (name, ref) in src_refs.slots # A slot absent from the linked module was dead (DCE'd or not imported under diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 8013f04c..e4035b5b 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -67,12 +67,11 @@ function load(obj::Vector{UInt8}, entry::String, refs::GPUCompiler.HostReference prefix = LLVM.get_prefix(lljit) add!(jd, LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)) - cells = Vector{UInt}(undef, length(refs.slots)) - roots = Any[] + relocations = GPUCompiler.resolved_relocations(refs) + cells = Vector{UInt}(undef, length(relocations.slots)) pairs = LLVM.API.LLVMOrcCSymbolMapPair[] - for (i, (name, ref)) in enumerate(refs.slots) - cells[i] = GPUCompiler.resolve_host_reference(ref) - ref isa GPUCompiler.JuliaValueRef && push!(roots, ref.value) + for (i, (name, value)) in enumerate(relocations.slots) + cells[i] = value symbol = LLVM.API.LLVMJITEvaluatedSymbol( reinterpret(UInt, pointer(cells, i)), LLVM.API.LLVMJITSymbolFlags( @@ -82,8 +81,12 @@ function load(obj::Vector{UInt8}, entry::String, refs::GPUCompiler.HostReference isempty(pairs) || LLVM.define(jd, LLVM.absolute_symbols(pairs)) add!(lljit, jd, MemoryBuffer(obj)) + for ((name, offset), value) in relocations.patches + addr = lookup(lljit, name) + unsafe_store!(Ptr{UInt}(pointer(addr) + offset), value) + end addr = lookup(lljit, entry) - return pointer(addr), (lljit, cells, roots) + return pointer(addr), (lljit, cells, relocations.roots) catch dispose(lljit) rethrow() diff --git a/test/native.jl b/test/native.jl index a2bbc841..fab4619a 100644 --- a/test/native.jl +++ b/test/native.jl @@ -776,6 +776,32 @@ end end @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.HostReferences()) + + # Interior relocations are applied after the object has been loaded. + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + patch_mod = parse(LLVM.Module, """ + @patched_box = externally_initialized global { i64, i64 } { i64 0, i64 42 } + + define i64 @patched_header() { + %value = load i64, $(ptr("i64")) getelementptr ({ i64, i64 }, $(ptr("{ i64, i64 }")) @patched_box, i32 0, i32 0) + ret i64 %value + }""") + triple!(patch_mod, GPUCompiler.llvm_triple(job.config.target)) + datalayout!(patch_mod, GPUCompiler.llvm_datalayout(job.config.target)) + patch_refs = GPUCompiler.HostReferences() + patch_refs.patches[("patched_box", 0)] = GPUCompiler.JuliaValueRef(Float64) + patch_obj = GPUCompiler.mcgen(job, patch_mod, LLVM.API.LLVMObjectFile) + patch_ptr, patch_keepalive = Native.load( + Vector{UInt8}(codeunits(patch_obj)), "patched_header", patch_refs) + try + GC.@preserve patch_keepalive begin + @test ccall(patch_ptr, UInt, ()) == + GPUCompiler.resolve_host_reference( + GPUCompiler.JuliaValueRef(Float64)) + end + finally + dispose(first(patch_keepalive)) + end end end end @@ -923,6 +949,54 @@ end only_needed=true) @test collect(keys(dest_refs.slots)) == ["used"] @test only(values(dest_refs.slots)).value === :used + + function patch_module(name, entry) + parse(LLVM.Module, """ + @$name = externally_initialized global { i64, i64 } { i64 0, i64 1 } + + define i64 @$entry() { + %value = load i64, $(ptr("i64")) getelementptr ({ i64, i64 }, $(ptr("{ i64, i64 }")) @$name, i32 0, i32 1) + ret i64 %value + }""") + end + patch_refs(name, value) = GPUCompiler.HostReferences( + Dict{String,GPUCompiler.HostReference}(), + Dict((name, 0) => GPUCompiler.JuliaValueRef(value))) + + # Identical patch identities merge; conflicting metadata does not. + dest = patch_module("patch", "first_patch") + dest_refs = patch_refs("patch", Float64) + src = patch_module("patch", "second_patch") + src_refs = patch_refs("patch", Float64) + GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) + @test collect(keys(dest_refs.patches)) == [("patch", 0)] + + dest = patch_module("patch", "first_patch") + dest_refs = patch_refs("patch", Float64) + src = patch_module("patch", "second_patch") + src_refs = patch_refs("patch", Int64) + @test_throws ErrorException GPUCompiler.link_with_host_references!( + dest, dest_refs, src, src_refs) + + # Metadata for patch globals not imported under `only_needed` is discarded. + dest = parse(LLVM.Module, """ + declare i64 @source_patch() + define i64 @entry_patch() { + %value = call i64 @source_patch() + ret i64 %value + }""") + src = patch_module("used_patch", "source_patch") + unused = GlobalVariable(src, LLVM.StructType([LLVM.Int64Type(), LLVM.Int64Type()]), + "unused_patch") + initializer!(unused, ConstantStruct(LLVM.Constant[ConstantInt(0), ConstantInt(1)])) + src_refs = GPUCompiler.HostReferences( + Dict{String,GPUCompiler.HostReference}(), + Dict(("used_patch", 0) => GPUCompiler.JuliaValueRef(Float64), + ("unused_patch", 0) => GPUCompiler.JuliaValueRef(Int64))) + dest_refs = GPUCompiler.HostReferences() + GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs; + only_needed=true) + @test collect(keys(dest_refs.patches)) == [("used_patch", 0)] end end From a80f05481be464e89f8d717fe8d303094f1462d3 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 20:26:19 +0200 Subject: [PATCH 24/45] Test patchable box visibility in PTX IR --- test/ptx.jl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/ptx.jl b/test/ptx.jl index aaf1190e..56099aa2 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -89,6 +89,19 @@ end end end +@testset "boxed header relocation" begin + mod = @eval module $(gensym()) + @noinline produce(cond::Bool, value::Int32) = cond ? value : 1.5 + function consume(cond::Bool, value::Int32) + x = produce(cond, value) + x isa Float64 && return x + return 0.0 + end + end + ir = sprint(io->PTX.code_llvm(io, mod.consume, Tuple{Bool,Int32}; dump_module=true)) + @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) +end + @testset "kernel functions" begin @testset "kernel argument attributes" begin mod = @eval module $(gensym()) From 663c014df97b12b0e9e7e09f7fd3a91aa598a668 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 20:29:26 +0200 Subject: [PATCH 25/45] Remove obsolete cache-wiping test references --- test/native.jl | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/native.jl b/test/native.jl index fab4619a..03a953a6 100644 --- a/test/native.jl +++ b/test/native.jl @@ -217,8 +217,6 @@ end @test new_res !== res @test new_res.asm === nothing - @test !isdefined(GPUCompiler, :session_dependent_jobs) - @test !isdefined(GPUCompiler, :wipe_session_dependent_results) end @testset "runtime cache invalidation" begin From df42587e2834c00151f4b868d40f1d44f5852134 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 21:00:15 +0200 Subject: [PATCH 26/45] Remove unused deprecation. --- src/deprecated.jl | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/deprecated.jl b/src/deprecated.jl index 7f41cfb4..7af67b40 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -274,9 +274,6 @@ end end # !HAS_INTEGRATED_CACHE -# Compatibility shim for backends using the former dynamic cache-wiping API. -mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing - ## Legacy `cached_compilation` (1.10+) From 39e532bdb9ddb771494613c4f4e8de44c4818859 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 16 Jul 2026 21:44:29 +0200 Subject: [PATCH 27/45] Adopt relocation terminology --- src/driver.jl | 26 +- src/interface.jl | 17 +- src/irgen.jl | 4 +- src/jlgen.jl | 2 +- src/mcgen.jl | 16 +- src/optim.jl | 16 +- src/relocation.jl | 563 +++++++++++++++++++++-------------------- src/rtlib.jl | 32 +-- src/validation.jl | 24 +- test/helpers/native.jl | 16 +- test/native.jl | 206 +++++++-------- test/ptx.jl | 2 +- 12 files changed, 465 insertions(+), 459 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index 9a308baa..167f5944 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -51,7 +51,7 @@ export compile Compile a `job` to one of the following formats as specified by the `target` argument: `:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. -The returned `host_references` metadata is final only for `:asm` and `:obj`, after runtime +The returned `relocations` metadata is final only for `:asm` and `:obj`, after runtime globals have been collected and the backend has lowered every live slot. The `:llvm` result still contains symbolic Julia-value slots and may contain raw `jl_*` runtime references. """ @@ -97,7 +97,7 @@ function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) else error("Unknown assembly format $output") end - asm, asm_meta = emit_asm(job, ir, ir_meta.host_references, format) + asm, asm_meta = emit_asm(job, ir, ir_meta.relocations, format) if output == :asm || output == :obj return asm, (; asm_meta..., ir_meta..., ir) @@ -184,7 +184,7 @@ const __llvm_initialized = Ref(false) end @tracepoint "IR generation" begin - ir, compiled, host_references = irgen(job) + ir, compiled, relocations = irgen(job) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -250,8 +250,8 @@ const __llvm_initialized = Ref(false) dyn_entry_fn = LLVM.name(dyn_meta.entry) merge!(compiled, dyn_meta.compiled) @assert context(dyn_ir) == context(ir) - link_with_host_references!(ir, host_references, dyn_ir, - dyn_meta.host_references) + link_relocatable!(ir, relocations, dyn_ir, + dyn_meta.relocations) changed = true dyn_entry_fn end @@ -294,7 +294,7 @@ const __llvm_initialized = Ref(false) if job.config.toplevel && job.config.libraries # load the runtime outside of a timing block (because it recurses into the compiler) if !uses_julia_runtime(job) - runtime, runtime_references = load_runtime(job) + runtime, runtime_relocs = load_runtime(job) end @tracepoint "Library linking" begin @@ -303,8 +303,8 @@ const __llvm_initialized = Ref(false) # GPU run-time library if !uses_julia_runtime(job) - @tracepoint "runtime library" link_with_host_references!( - ir, host_references, runtime, runtime_references; only_needed=true) + @tracepoint "runtime library" link_relocatable!( + ir, relocations, runtime, runtime_relocs; only_needed=true) end end end @@ -339,7 +339,7 @@ const __llvm_initialized = Ref(false) if job.config.optimize @tracepoint "optimization" begin - optimize!(job, ir, host_references; job.config.opt_level) + optimize!(job, ir, relocations; job.config.opt_level) # deferred codegen has some special optimization requirements, # which also need to happen _after_ regular optimization. @@ -404,7 +404,7 @@ const __llvm_initialized = Ref(false) if job.config.toplevel && job.config.validate @tracepoint "validation" begin - check_ir(job, ir, host_references) + check_ir(job, ir, relocations) end end @@ -412,18 +412,18 @@ const __llvm_initialized = Ref(false) @tracepoint "verification" verify(ir) end - return ir, (; entry, compiled, host_references) + return ir, (; entry, compiled, relocations) end @locked function emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, - refs::HostReferences, format::LLVM.API.LLVMCodeGenFileType) + relocs::Relocations, format::LLVM.API.LLVMCodeGenFileType) # NOTE: strip after validation to get better errors if job.config.strip @tracepoint "Debug info removal" strip_debuginfo!(ir) end @tracepoint "LLVM back-end" begin - @tracepoint "preparation" prepare_execution!(job, ir, refs) + @tracepoint "preparation" prepare_execution!(job, ir, relocs) code = @tracepoint "machine-code generation" mcgen(job, ir, format) end diff --git a/src/interface.jl b/src/interface.jl index a042f097..a715a207 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -429,13 +429,12 @@ the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock """ function cached_results end -# Host-reference types and the backend lowering hook live in relocation.jl. +# Relocation types and the backend lowering hook live in relocation.jl. # # Job results attached to CodeInstances are serialized into package images. Store only # session-portable data there. Generated code is portable iff `supports_relocatable_ir()` -# and the backend preserves host references with definition- or declaration-based lowering, -# then patches them at load time. Backends using eager lowering must keep generated code in a -# session-local cache or recompile it. +# and the backend preserves relocations with patchable- or imported-symbol lowering. Backends +# using baked lowering must keep generated code in a session-local cache or recompile it. @static if HAS_INTEGRATED_CACHE """ @@ -513,11 +512,11 @@ end end # HAS_INTEGRATED_CACHE -@public GPUCompilerCacheToken, cache_owner, cached_results, supports_relocatable_ir -@public JuliaValueRef, CGlobalRef, HostReference, HostReferences -@public lower_host_references!, emit_host_reference_definitions! -@public emit_host_reference_declarations! -@public resolve_host_reference, resolved_relocations +@public JuliaValueRef, CGlobalRef, RelocationTarget, Relocations +@public lower_relocations!, bake_relocations! +@public emit_patchable_relocations!, emit_imported_relocations! +@public resolve_relocation_target, resolved_relocations, supports_relocatable_ir +@public GPUCompilerCacheToken, cache_owner, cached_results # the method table to use # diff --git a/src/irgen.jl b/src/irgen.jl index 702c0a3e..648459be 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -2,7 +2,7 @@ function irgen(@nospecialize(job::CompilerJob)) mod, compiled, gv_to_value = @tracepoint "emission" compile_method_instance(job) - host_references = collect_julia_value_references!(mod, gv_to_value) + relocations = collect_julia_value_relocations!(mod, gv_to_value) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -150,7 +150,7 @@ function irgen(@nospecialize(job::CompilerJob)) lower_alloca!(job, mod) end - return mod, compiled, host_references + return mod, compiled, relocations end diff --git a/src/jlgen.jl b/src/jlgen.jl index e4d5e7c6..ad0cebf3 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -600,7 +600,7 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) # Strip the initializers so the IR we hand back is session-portable # (on 1.12 `jl_emit_native_impl` bakes pointers via # `literal_static_pointer_val`; on 1.13+ Julia nulls them itself); - # host-reference lowering writes the session-current value in. Demote + # relocation lowering writes the session-current value in. Demote # each GV to an external *declaration* rather than # giving it a null initializer: an internal global initialized to null # is fair game for optimization passes that run before relocation diff --git a/src/mcgen.jl b/src/mcgen.jl index 9b754182..fdb96349 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -15,21 +15,21 @@ end # Final preparations for the module to be compiled to machine code. These passes should not # be run when e.g. compiling to write to disk. function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences=HostReferences()) - # Clean up first so only live references get slots and get lowered. + relocs::Relocations=Relocations()) + # Clean up first so only live relocations get lowered. run_cleanup_pipeline!(job, mod) - prune_dead_host_references!(mod, refs) + prune_dead_relocations!(mod, relocs) - collect_runtime_global_references!(mod, refs) - lower_host_references!(job, mod, refs) + collect_cglobal_relocations!(mod, relocs) + lower_relocations!(job, mod, relocs) # Fold constants exposed by eager lowering, and discard slots made dead by either # lowering strategy. run_cleanup_pipeline!(job, mod) - prune_dead_host_references!(mod, refs) + prune_dead_relocations!(mod, relocs) - has_unresolved_runtime_global_loads(mod, refs) && - error("Unresolved Julia runtime global load after host-reference lowering") + has_unresolved_cglobal_loads(mod, relocs) && + error("Unresolved cglobal load after relocation lowering") return end diff --git a/src/optim.jl b/src/optim.jl index 5cbae38c..02bf7c77 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -50,7 +50,7 @@ function aggressiveinstcombine_pass(@nospecialize(job::CompilerJob)) end function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences; opt_level=2) + relocs::Relocations; opt_level=2) tm = llvm_machine(job.config.target) tti = llvm_targetinfo(job.config.target) @@ -60,7 +60,7 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module, register!(pb, GPULowerCPUFeaturesPass(job)) register!(pb, GPULowerPTLSPass(job)) register!(pb, GPULowerGCFramePass(job)) - register!(pb, GPULinkRuntimePass(job, refs)) + register!(pb, GPULinkRuntimePass(job, relocs)) register!(pb, GPULinkLibrariesPass(job)) register!(pb, GPUFinishRuntimeIntrinsicsPass(job)) register!(pb, AddKernelStatePass(job)) @@ -327,7 +327,7 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op end if job.config.libraries # The pass builder resolves this name to the registered instance, which - # captures the HostReferences object owned by optimize!. + # captures the Relocations object owned by optimize!. add!(mpm, "GPULinkRuntime") add!(mpm, GPULinkLibrariesPass(job)) add!(mpm, GPUFinishRuntimeIntrinsicsPass(job)) @@ -478,7 +478,7 @@ GPULowerCPUFeaturesPass(job) = NewPMModulePass("GPULowerCPUFeatures", CPUFeature struct LinkRuntime job::CompilerJob - refs::HostReferences + relocs::Relocations end function (self::LinkRuntime)(mod::LLVM.Module) self.job.config.libraries || return false @@ -487,16 +487,16 @@ function (self::LinkRuntime)(mod::LLVM.Module) # GC lowering can introduce new calls to GPU runtime functions after the runtime # was linked initially. Link again now so those calls resolve to definitions before # later intrinsic-lowering passes inspect or rewrite the runtime call graph. - runtime, runtime_refs = load_runtime(self.job) + runtime, runtime_relocs = load_runtime(self.job) # `RemoveNIPass` stripped non-integral address spaces from `mod`'s datalayout, but the # cached runtime kept them; align it (as with target libraries) to avoid a warning. triple!(runtime, triple(mod)) datalayout!(runtime, datalayout(mod)) - link_with_host_references!(mod, self.refs, runtime, runtime_refs; only_needed=true) + link_relocatable!(mod, self.relocs, runtime, runtime_relocs; only_needed=true) return true end -GPULinkRuntimePass(job, refs::HostReferences) = - NewPMModulePass("GPULinkRuntime", LinkRuntime(job, refs)) +GPULinkRuntimePass(job, relocs::Relocations) = + NewPMModulePass("GPULinkRuntime", LinkRuntime(job, relocs)) struct LinkLibraries job::CompilerJob diff --git a/src/relocation.jl b/src/relocation.jl index 6c26f9fa..5a68221b 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -1,26 +1,35 @@ -# Host-reference relocation lifecycle +# Relocation model +# +# A relocation is a site => target pair. Targets are Julia object identities +# (`JuliaValueRef`, like codegen's identity-keyed `global_targets`) or named libjulia +# globals (`CGlobalRef`, like JuliaVariables). Sites are either dedicated word-sized +# `slots` (like `jl_sysimg_gvars`) or word-sized `interior` locations inside defined +# globals (like `gctags_list`/`relocs_list`). Slots can be baked, patched, or imported; +# interior sites can only be baked or patched. # # produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) # -# Julia values produce two relocation kinds: word-sized declaration slots and patches at a -# byte offset inside a defined global. Runtime globals produce slots. Names are globally -# unique and assigned once, so linking can merge IR and metadata without renaming. +# Lowering strategy Loader contract Julia analog +# `bake_relocations!` none; current-session words baked JIT address folding +# `emit_patchable_relocations!` patch named globals after load sysimage gvars +# `emit_imported_relocations!` import slots before load; patch ORC absoluteSymbols +# interior sites afterward # -# The two producers are `collect_julia_value_references!` during IR generation and -# `collect_runtime_global_references!` immediately before backend lowering. +# The producers are `collect_julia_value_relocations!` during IR generation and +# `collect_cglobal_relocations!` immediately before backend lowering. Names are globally +# unique and assigned once, so linking can merge IR and metadata without renaming. # # Generated code is portable only when `supports_relocatable_ir()` and the backend preserves -# these relocations for its loader. The default eager lowering bakes current-session values. -# -# This mirrors Julia's own mechanisms: codegen's identity-keyed global_targets slots, the -# sysimage jl_gvars table patched by jl_update_all_gvars, and JIT absoluteSymbols definitions. +# these relocations for its loader. The default lowering bakes current-session values. + +## Targets """ JuliaValueRef(value) A Julia value with a stable address (a heap object, symbol, or singleton), used as the -serializable identity of a host reference. Resolve it in the active session with -[`resolve_host_reference`](@ref); a loader that writes the resulting address into device +serializable identity of a relocation target. Resolve it in the active session with +[`resolve_relocation_target`](@ref); a loader that writes the resulting address into device storage must keep `value` rooted for as long as that storage remains reachable. """ struct JuliaValueRef @@ -44,92 +53,124 @@ struct CGlobalRef end """ - HostReference + RelocationTarget -A serializable source for a host-derived word: either a [`JuliaValueRef`](@ref) or a +A serializable target for a relocated word: either a [`JuliaValueRef`](@ref) or a [`CGlobalRef`](@ref). """ -const HostReference = Union{JuliaValueRef,CGlobalRef} - -""" - HostReferences(slots, patches) - -Host-reference metadata accompanying a module. `slots` maps word-sized declaration symbols to -the word resolved by a loader. `patches` maps a global name and byte offset to a word patched -after loading. -""" -struct HostReferences - slots::Dict{String,HostReference} - patches::Dict{Tuple{String,Int},HostReference} -end +const RelocationTarget = Union{JuliaValueRef,CGlobalRef} -HostReferences() = HostReferences(Dict{String,HostReference}(), - Dict{Tuple{String,Int},HostReference}()) - -same_host_reference(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value -same_host_reference(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol -same_host_reference(::HostReference, ::HostReference) = false +same_relocation_target(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value +same_relocation_target(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol +same_relocation_target(::RelocationTarget, ::RelocationTarget) = false """ - resolve_host_reference(ref) -> UInt + resolve_relocation_target(target) -> UInt -Resolve a host reference to its word in the current Julia process. +Resolve a relocation target to its word in the current Julia process. """ -function resolve_host_reference(ref::JuliaValueRef) - box = Any[ref.value] +function resolve_relocation_target(target::JuliaValueRef) + box = Any[target.value] GC.@preserve box begin return unsafe_load(Base.unsafe_convert(Ptr{UInt}, pointer(box))) end end -function resolve_host_reference(ref::CGlobalRef) - address = ccall(:jl_cglobal, Any, (Any, Any), String(ref.symbol), UInt) +function resolve_relocation_target(target::CGlobalRef) + address = ccall(:jl_cglobal, Any, (Any, Any), String(target.symbol), UInt) return unsafe_load(address) end +## The table + +""" + Relocations(slots, interior) + +Relocation metadata accompanying a module. `slots` maps dedicated word-sized global names +to targets. `interior` maps a defined global name and byte offset to targets. See +[`resolved_relocations`](@ref) for resolving both site kinds for a loader. +""" +struct Relocations + slots::Dict{String,RelocationTarget} + interior::Dict{Tuple{String,Int},RelocationTarget} +end + +Relocations() = Relocations(Dict{String,RelocationTarget}(), + Dict{Tuple{String,Int},RelocationTarget}()) + """ - resolved_relocations(refs) + resolved_relocations(relocs) -Resolve relocation metadata for a loader. Returns resolved `slots`, resolved `patches`, and +Resolve relocation metadata for a loader. Returns resolved `slots`, resolved `interior`, and Julia `roots` that must stay alive while the loaded code can access them. """ -function resolved_relocations(refs::HostReferences) +function resolved_relocations(relocs::Relocations) slots = Pair{String,UInt}[] - patches = Pair{Tuple{String,Int},UInt}[] + interior = Pair{Tuple{String,Int},UInt}[] roots = Any[] - for (name, ref) in refs.slots - push!(slots, name => resolve_host_reference(ref)) + for (name, ref) in relocs.slots + push!(slots, name => resolve_relocation_target(ref)) ref isa JuliaValueRef && push!(roots, ref.value) end - for (key, ref) in refs.patches - push!(patches, key => resolve_host_reference(ref)) + for (key, ref) in relocs.interior + push!(interior, key => resolve_relocation_target(ref)) ref isa JuliaValueRef && push!(roots, ref.value) end - return (; slots, patches, roots) + return (; slots, interior, roots) end -""" - lower_host_references!(job, mod, refs) +relocation_word_type() = LLVM.IntType(8sizeof(UInt)) -Backend hook for lowering live host references before object emission. +function check_slot_size(mod::LLVM.Module, gv::GlobalVariable, name::String) + size = abi_size(datalayout(mod), global_value_type(gv)) + size == sizeof(UInt) || + error("Relocation slot '$name' has size $size, expected $(sizeof(UInt))") + return +end -The default implementation resolves them in the current Julia process, making the result -session-dependent. Loaders may instead emit module-owned definitions that are patched after -loading, or external declarations that are defined before loading. +function slot_initializer(gv::GlobalVariable, value::UInt) + T = global_value_type(gv) + if T isa LLVM.PointerType + return const_inttoptr(ConstantInt(UInt64(value)), T) + elseif T isa LLVM.IntegerType && width(T) == 8sizeof(UInt) + return ConstantInt(T, value) + end + error("Relocation slot '$(LLVM.name(gv))' has unsupported LLVM type $T") +end -Backends using eager lowering must not persist generated code in `cached_results`. Generated -code is session-portable only when [`supports_relocatable_ir`](@ref) and the backend preserves -all slots and patches with definition- or declaration-based lowering, then applies them at -load time. -""" -function lower_host_references!(@nospecialize(job::CompilerJob), mod::LLVM.Module, - refs::HostReferences) - resolve_host_references!(mod, refs) +function foreach_slot_relocation(f, mod::LLVM.Module, relocs::Relocations) + mod_gvs = globals(mod) + for (name, ref) in relocs.slots + haskey(mod_gvs, name) || error("Missing relocation slot '$name'") + gv = mod_gvs[name] + check_slot_size(mod, gv, name) + f(name, gv, ref) + end + return +end + +function foreach_interior_relocation(f, mod::LLVM.Module, relocs::Relocations) + mod_gvs = globals(mod) + for ((name, offset), ref) in relocs.interior + haskey(mod_gvs, name) || error("Missing interior relocation global '$name'") + gv = mod_gvs[name] + init = initializer(gv) + init === nothing && error("Interior relocation global '$name' has no initializer") + T = value_type(init) + T isa LLVM.StructType || + error("Interior relocation global '$name' has non-struct initializer $T") + size = abi_size(datalayout(mod), T) + 0 <= offset && offset + sizeof(UInt) <= size || + error("Interior relocation '$name+$offset' is outside its $size-byte global") + f(name, offset, gv, ref) + end + return end +## Producers -function collect_julia_value_references!(mod::LLVM.Module, +function collect_julia_value_relocations!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - refs = HostReferences() + relocs = Relocations() mod_gvs = globals(mod) for (name, init) in gv_to_value haskey(mod_gvs, name) || continue @@ -145,16 +186,16 @@ function collect_julia_value_references!(mod::LLVM.Module, init == C_NULL && error("Missing Julia object for global '$name'") obj = Base.unsafe_pointer_to_objref(init) if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) - val = materialize_box!(mod, refs, gv, obj, init) + val = materialize_box!(mod, relocs, gv, obj, init) initializer!(gv, val) linkage!(gv, LLVM.API.LLVMPrivateLinkage) else - host_reference_slot_size(mod, gv, name) + check_slot_size(mod, gv, name) slot_name = safe_name(name) * "_" * string(objectid(obj); base=16) LLVM.name!(gv, slot_name) LLVM.name(gv) == slot_name || - error("Host reference slot name '$slot_name' is already in use") - refs.slots[slot_name] = JuliaValueRef(obj) + error("Relocation slot name '$slot_name' is already in use") + relocs.slots[slot_name] = JuliaValueRef(obj) end end @@ -169,196 +210,17 @@ function collect_julia_value_references!(mod::LLVM.Module, end init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) - val = materialize_box!(mod, refs, gv, obj, init) + val = materialize_box!(mod, relocs, gv, obj, init) initializer!(gv, val) constant!(gv, true) linkage!(gv, LLVM.API.LLVMPrivateLinkage) end - return refs -end - -host_reference_word_type() = LLVM.IntType(8sizeof(UInt)) - -function host_reference_slot_size(mod::LLVM.Module, gv::GlobalVariable, name::String) - size = abi_size(datalayout(mod), global_value_type(gv)) - size == sizeof(UInt) || - error("Host reference slot '$name' has size $size, expected $(sizeof(UInt))") - return -end - -function host_reference_slot_initializer(gv::GlobalVariable, value::UInt) - T = global_value_type(gv) - if T isa LLVM.PointerType - return const_inttoptr(ConstantInt(UInt64(value)), T) - elseif T isa LLVM.IntegerType && width(T) == 8sizeof(UInt) - return ConstantInt(T, value) - end - error("Host reference slot '$(LLVM.name(gv))' has unsupported LLVM type $T") + return relocs end -function foreach_host_reference_slot(f, mod::LLVM.Module, refs::HostReferences) - mod_gvs = globals(mod) - for (name, ref) in refs.slots - haskey(mod_gvs, name) || error("Missing host reference slot '$name'") - gv = mod_gvs[name] - host_reference_slot_size(mod, gv, name) - f(name, gv, ref) - end - return -end - -function foreach_host_reference_patch(f, mod::LLVM.Module, refs::HostReferences) - mod_gvs = globals(mod) - for ((name, offset), ref) in refs.patches - haskey(mod_gvs, name) || error("Missing host reference patch global '$name'") - gv = mod_gvs[name] - init = initializer(gv) - init === nothing && error("Host reference patch global '$name' has no initializer") - T = value_type(init) - T isa LLVM.StructType || - error("Host reference patch global '$name' has non-struct initializer $T") - size = abi_size(datalayout(mod), T) - 0 <= offset && offset + sizeof(UInt) <= size || - error("Host reference patch '$name+$offset' is outside its $size-byte global") - f(name, offset, gv, ref) - end - return -end - -function prune_dead_host_references!(mod::LLVM.Module, refs::HostReferences) - mod_gvs = globals(mod) - for name in collect(keys(refs.slots)) - haskey(mod_gvs, name) || delete!(refs.slots, name) - end - for ((name, offset), _) in collect(refs.patches) - if !haskey(mod_gvs, name) - delete!(refs.patches, (name, offset)) - elseif isempty(uses(mod_gvs[name])) - delete!(refs.patches, (name, offset)) - erase!(mod_gvs[name]) - end - end - return -end - -function resolve_host_references!(mod::LLVM.Module, refs::HostReferences) - foreach_host_reference_slot(mod, refs) do name, gv, ref - value = resolve_host_reference(ref) - val = host_reference_slot_initializer(gv, value) - initializer!(gv, val) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - constant!(gv, true) - end - foreach_host_reference_patch(mod, refs) do name, offset, gv, ref - init = initializer(gv) - T = value_type(init)::LLVM.StructType - idx = Int(element_at(datalayout(mod), T, offset)) + 1 - fields = LLVM.Constant[operands(init)...] - fields[idx] = ConstantInt(value_type(fields[idx]), resolve_host_reference(ref)) - initializer!(gv, ConstantStruct(T, fields)) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - extinit!(gv, false) - constant!(gv, true) - unnamed_addr!(gv, true) - end - empty!(refs.slots) - empty!(refs.patches) - return -end - -""" - emit_host_reference_definitions!(mod, refs) - -Emit host-reference slots as writable, null-initialized definitions. The loader must patch each -definition and interior relocation after loading the object. This requires a per-object symbol -namespace. -""" -function emit_host_reference_definitions!(mod::LLVM.Module, refs::HostReferences) - slots = GlobalVariable[] - foreach_host_reference_slot(mod, refs) do _, gv, _ - initializer!(gv, null(global_value_type(gv))) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, true) - push!(slots, gv) - end - foreach_host_reference_patch(mod, refs) do _, _, gv, _ - push!(slots, gv) - end - isempty(slots) || set_used!(mod, slots...) - return -end - -""" - emit_host_reference_declarations!(mod, refs) - -Prepare host references for a loader that defines symbols before loading an object. - -Every slot remains an external word-sized declaration. The loader must define each symbol to -point at a cell containing [`resolve_host_reference`](@ref), and keep the cell and any -referenced Julia value alive while the code is executable. Interior patches cannot be defined -before load and must always be written afterward. -""" -function emit_host_reference_declarations!(mod::LLVM.Module, refs::HostReferences) - used = GlobalVariable[] - foreach_host_reference_slot(mod, refs) do name, gv, _ - isdeclaration(gv) || error("Host reference slot '$name' must be a declaration") - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, false) - end - foreach_host_reference_patch(mod, refs) do _, _, gv, _ - push!(used, gv) - end - isempty(used) || set_used!(mod, used...) - return -end - -function link_with_host_references!(dest_mod::LLVM.Module, dest_refs::HostReferences, - src_mod::LLVM.Module, src_refs::HostReferences; - only_needed=false) - # Patch globals are definitions, unlike slots. Make an identical source definition a - # declaration so LLVM can resolve it to the destination definition while linking. - for (key, ref) in src_refs.patches - existing = get(dest_refs.patches, key, nothing) - existing === nothing && continue - name, offset = key - same_host_reference(existing, ref) || - error("Host reference patch '$name+$offset' refers to conflicting values") - haskey(globals(dest_mod), name) || - error("Missing destination host reference patch global '$name'") - haskey(globals(src_mod), name) || - error("Missing source host reference patch global '$name'") - gv = globals(src_mod)[name] - initializer!(gv, nothing) - extinit!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - end - - link!(dest_mod, src_mod; only_needed) - for (name, ref) in src_refs.slots - # A slot absent from the linked module was dead (DCE'd or not imported under - # `only_needed`); its relocation dies with it. - haskey(globals(dest_mod), name) || continue - existing = get(dest_refs.slots, name, nothing) - existing === nothing || same_host_reference(existing, ref) || - error("Host reference slot '$name' refers to conflicting values") - dest_refs.slots[name] = ref - end - for ((name, offset), ref) in src_refs.patches - haskey(globals(dest_mod), name) || continue - key = (name, offset) - existing = get(dest_refs.patches, key, nothing) - existing === nothing || same_host_reference(existing, ref) || - error("Host reference patch '$name+$offset' refers to conflicting values") - dest_refs.patches[key] = ref - end - return -end - -# emit a device-resident constant replica of the box holding `obj`; returns -# the constant to store in the slot, and the (gcbits-masked) header word -function materialize_box!(mod::LLVM.Module, refs::HostReferences, gv::GlobalVariable, +# Emit a device-resident constant replica of the box holding `obj` and return +# the constant to store in its slot. Any relocatable header is recorded in `relocs`. +function materialize_box!(mod::LLVM.Module, relocs::Relocations, gv::GlobalVariable, @nospecialize(obj), init::Ptr{Cvoid}) @assert isbitstype(typeof(obj)) && sizeof(obj) > 0 @@ -393,7 +255,7 @@ function materialize_box!(mod::LLVM.Module, refs::HostReferences, gv::GlobalVari safe_name(LLVM.name(gv)) * "_box" end box = GlobalVariable(mod, boxty, box_name) - LLVM.name(box) == box_name || error("Host reference patch global '$box_name' is already in use") + LLVM.name(box) == box_name || error("Interior relocation global '$box_name' is already in use") initializer!(box, boxinit) alignment!(box, 16) if patch_header @@ -401,7 +263,7 @@ function materialize_box!(mod::LLVM.Module, refs::HostReferences, gv::GlobalVari linkage!(box, LLVM.API.LLVMExternalLinkage) extinit!(box, true) offset = Int(offsetof(datalayout(mod), boxty, header_idx)) - refs.patches[(box_name, offset)] = JuliaValueRef(typeof(obj)) + relocs.interior[(box_name, offset)] = JuliaValueRef(typeof(obj)) else constant!(box, true) linkage!(box, LLVM.API.LLVMPrivateLinkage) @@ -415,37 +277,30 @@ function materialize_box!(mod::LLVM.Module, refs::HostReferences, gv::GlobalVari return val end - -# some Julia code contains references to objects in the CPU run-time, -# without actually using the contents or functionality of those objects. -# -# prime example are type tags, which reference the address of the allocated type. -# since those references are ephemeral, we can't eagerly resolve and emit them in the IR, -# but at the same time the GPU can't resolve them at run-time. -# -# this collection performs that resolution at object-emission time. -function is_runtime_global_candidate(value, refs::HostReferences) +# Some Julia code loads words from libjulia C globals, for example type tags. Record those +# loads as slot relocations immediately before object emission. +function is_cglobal_candidate(value, relocs::Relocations) name = LLVM.name(value) - value isa LLVM.GlobalVariable && haskey(refs.slots, name) && return false + value isa LLVM.GlobalVariable && haskey(relocs.slots, name) && return false isdeclaration(value) || return false value isa LLVM.Function && LLVM.isintrinsic(value) && return false return startswith(name, "jl_") end -function collect_runtime_global_references!(mod::LLVM.Module, refs::HostReferences) +function collect_cglobal_relocations!(mod::LLVM.Module, relocs::Relocations) changed = false for f in [collect(functions(mod)); collect(globals(mod))] - is_runtime_global_candidate(f, refs) || continue + is_cglobal_candidate(f, relocs) || continue fn = LLVM.name(f) slot = nothing - function runtime_global_slot() + function cglobal_slot() if slot === nothing name = "gpu_" * fn - slot = GlobalVariable(mod, host_reference_word_type(), name) + slot = GlobalVariable(mod, relocation_word_type(), name) LLVM.name(slot) == name || - error("Julia runtime global slot name '$name' is already in use") - refs.slots[name] = CGlobalRef(Symbol(fn)) + error("cglobal slot name '$name' is already in use") + relocs.slots[name] = CGlobalRef(Symbol(fn)) end slot end @@ -461,12 +316,12 @@ function collect_runtime_global_references!(mod::LLVM.Module, refs::HostReferenc T = value_type(val) if !(T isa LLVM.PointerType || (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) - error("Unsupported Julia runtime global '$fn' load of LLVM type $T") + error("Unsupported cglobal '$fn' load of LLVM type $T") end @dispose builder=IRBuilder() begin position!(builder, val) - replacement = load!(builder, host_reference_word_type(), - runtime_global_slot()) + replacement = load!(builder, relocation_word_type(), + cglobal_slot()) if T isa LLVM.PointerType replacement = inttoptr!(builder, replacement, T) end @@ -485,7 +340,7 @@ function collect_runtime_global_references!(mod::LLVM.Module, refs::HostReferenc return changed end -function has_unresolved_runtime_global_loads(mod::LLVM.Module, refs::HostReferences) +function has_unresolved_cglobal_loads(mod::LLVM.Module, relocs::Relocations) function has_load(value) for use in uses(value) val = user(use) @@ -496,14 +351,166 @@ function has_unresolved_runtime_global_loads(mod::LLVM.Module, refs::HostReferen end for value in [collect(functions(mod)); collect(globals(mod))] - is_runtime_global_candidate(value, refs) || continue + is_cglobal_candidate(value, relocs) || continue has_load(value) && return true end return false end +## Bookkeeping + +function link_relocatable!(dest_mod::LLVM.Module, dest_relocs::Relocations, + src_mod::LLVM.Module, src_relocs::Relocations; + only_needed=false) + # Interior relocation globals are definitions, unlike slots. Make an identical source + # definition a declaration so LLVM can resolve it to the destination while linking. + for (key, ref) in src_relocs.interior + existing = get(dest_relocs.interior, key, nothing) + existing === nothing && continue + name, offset = key + same_relocation_target(existing, ref) || + error("Interior relocation '$name+$offset' refers to conflicting values") + haskey(globals(dest_mod), name) || + error("Missing destination interior relocation global '$name'") + haskey(globals(src_mod), name) || + error("Missing source interior relocation global '$name'") + gv = globals(src_mod)[name] + initializer!(gv, nothing) + extinit!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + end + + link!(dest_mod, src_mod; only_needed) + for (name, ref) in src_relocs.slots + # A slot absent from the linked module was dead (DCE'd or not imported under + # `only_needed`); its relocation dies with it. + haskey(globals(dest_mod), name) || continue + existing = get(dest_relocs.slots, name, nothing) + existing === nothing || same_relocation_target(existing, ref) || + error("Relocation slot '$name' refers to conflicting values") + dest_relocs.slots[name] = ref + end + for ((name, offset), ref) in src_relocs.interior + haskey(globals(dest_mod), name) || continue + key = (name, offset) + existing = get(dest_relocs.interior, key, nothing) + existing === nothing || same_relocation_target(existing, ref) || + error("Interior relocation '$name+$offset' refers to conflicting values") + dest_relocs.interior[key] = ref + end + return +end + +function prune_dead_relocations!(mod::LLVM.Module, relocs::Relocations) + mod_gvs = globals(mod) + for name in collect(keys(relocs.slots)) + haskey(mod_gvs, name) || delete!(relocs.slots, name) + end + for ((name, offset), _) in collect(relocs.interior) + if !haskey(mod_gvs, name) + delete!(relocs.interior, (name, offset)) + elseif isempty(uses(mod_gvs[name])) + delete!(relocs.interior, (name, offset)) + erase!(mod_gvs[name]) + end + end + return +end + +## Lowering + +""" + lower_relocations!(job, mod, relocs) + +Backend hook for lowering live relocations before object emission. + +The default implementation bakes them into the module using the current Julia process. +Backends may instead emit patchable globals or imported slot symbols for their loaders. + +Backends using baked lowering must not persist generated code in `cached_results`. Generated +code is session-portable only when [`supports_relocatable_ir`](@ref) and the backend preserves +all slot and interior relocations for its loader. +""" +function lower_relocations!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + relocs::Relocations) + bake_relocations!(mod, relocs) +end + +function bake_relocations!(mod::LLVM.Module, relocs::Relocations) + foreach_slot_relocation(mod, relocs) do name, gv, ref + value = resolve_relocation_target(ref) + val = slot_initializer(gv, value) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + constant!(gv, true) + end + foreach_interior_relocation(mod, relocs) do name, offset, gv, ref + init = initializer(gv) + T = value_type(init)::LLVM.StructType + idx = Int(element_at(datalayout(mod), T, offset)) + 1 + fields = LLVM.Constant[operands(init)...] + fields[idx] = ConstantInt(value_type(fields[idx]), resolve_relocation_target(ref)) + initializer!(gv, ConstantStruct(T, fields)) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + extinit!(gv, false) + constant!(gv, true) + unnamed_addr!(gv, true) + end + empty!(relocs.slots) + empty!(relocs.interior) + return +end + +""" + emit_patchable_relocations!(mod, relocs) + +Emit slots as writable, null-initialized definitions. The loader must patch each definition +and interior relocation after loading the object. This requires a per-object symbol namespace. +""" +function emit_patchable_relocations!(mod::LLVM.Module, relocs::Relocations) + used = GlobalVariable[] + foreach_slot_relocation(mod, relocs) do _, gv, _ + initializer!(gv, null(global_value_type(gv))) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, true) + push!(used, gv) + end + foreach_interior_relocation(mod, relocs) do _, _, gv, _ + push!(used, gv) + end + isempty(used) || set_used!(mod, used...) + return +end + +""" + emit_imported_relocations!(mod, relocs) + +Prepare relocations for a loader that imports slot symbols before loading an object. + +Every slot remains an external word-sized declaration. The loader must define each symbol to +point at a cell containing [`resolve_relocation_target`](@ref), and keep the cell and any +referenced Julia value alive while the code is executable. Interior sites cannot be imported +before load and must always be written afterward. +""" +function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) + used = GlobalVariable[] + foreach_slot_relocation(mod, relocs) do name, gv, _ + isdeclaration(gv) || error("Relocation slot '$name' must be a declaration") + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + end + foreach_interior_relocation(mod, relocs) do _, _, gv, _ + push!(used, gv) + end + isempty(used) || set_used!(mod, used...) + return +end + +## Introspection -function referenced_object(value, refs::HostReferences) +function referenced_object(value, relocs::Relocations) # This is best-effort: optimized shapes fall back to the unknown-binding error path. while value isa ConstantExpr && opcode(value) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) @@ -516,7 +523,7 @@ function referenced_object(value, refs::HostReferences) source = first(operands(source)) end if source isa GlobalVariable - ref = get(refs.slots, LLVM.name(source), nothing) + ref = get(relocs.slots, LLVM.name(source), nothing) ref isa JuliaValueRef && return Some(ref.value) end elseif value isa ConstantExpr && opcode(value) == LLVM.API.LLVMIntToPtr diff --git a/src/rtlib.jl b/src/rtlib.jl index 912d6aa1..8e619c4e 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -68,14 +68,14 @@ end # entirely. On 1.10 it is cached for the duration of the session. mutable struct RuntimeFunctionResults bitcode::Union{Nothing,Vector{UInt8}} - host_references::HostReferences - RuntimeFunctionResults() = new(nothing, HostReferences()) + relocations::Relocations + RuntimeFunctionResults() = new(nothing, Relocations()) end # Compile a single runtime function and link it into `mod`. The renamed bitcode is # memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache # below additionally avoids repeating the parse-and-link work within a session. -function emit_function!(mod, refs::HostReferences, config::CompilerConfig, +function emit_function!(mod, relocs::Relocations, config::CompilerConfig, source::MethodInstance, method, world::UInt) name = method.llvm_name rt_job = CompilerJob(source, config, world) @@ -84,9 +84,9 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, # inference itself. ci, res = runtime_function_results(rt_job) if res !== nothing && res.bitcode !== nothing - link_with_host_references!(mod, refs, - parse(LLVM.Module, MemoryBuffer(res.bitcode)), - res.host_references) + link_relocatable!(mod, relocs, + parse(LLVM.Module, MemoryBuffer(res.bitcode)), + res.relocations) ci === nothing && (ci = runtime_code_instance(rt_job)) return ci::CodeInstance end @@ -100,7 +100,7 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - prune_dead_host_references!(new_mod, meta.host_references) + prune_dead_relocations!(new_mod, meta.relocations) # rename to the final `gpu_*` name on the per-function module, so the cached bitcode # is immediately link-ready (no per-session rename pass on a cache hit). @@ -118,10 +118,10 @@ function emit_function!(mod, refs::HostReferences, config::CompilerConfig, if supports_relocatable_ir() res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) res.bitcode = take!(io) - res.host_references = meta.host_references + res.relocations = meta.relocations end - link_with_host_references!(mod, refs, new_mod, meta.host_references) + link_relocatable!(mod, relocs, new_mod, meta.relocations) return ci::CodeInstance end @@ -170,14 +170,14 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) mod = LLVM.Module("GPUCompiler run-time library") sources = MethodInstance[] code_instances = CodeInstance[] - refs = HostReferences() + relocs = Relocations() for method in values(Runtime.methods) resolved = runtime_method_instance(job, method) resolved === nothing && continue source = resolved push!(sources, source) - push!(code_instances, emit_function!(mod, refs, config, source, method, job.world)) + push!(code_instances, emit_function!(mod, relocs, config, source, method, job.world)) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -185,7 +185,7 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) # removes Julia address spaces, which would then lead to type mismatches when using # functions from the runtime library from IR that has not been stripped of AS info. - return mod, sources, code_instances, refs + return mod, sources, code_instances, relocs end # Session-local cache of assembled runtime libraries, keyed by @@ -204,7 +204,7 @@ mutable struct RuntimeLibrary sources::Vector{MethodInstance} code_instances::Vector{CodeInstance} validated_world::UInt - host_references::HostReferences + relocations::Relocations end function runtime_library_valid(lib::RuntimeLibrary, @nospecialize(job::CompilerJob)) @@ -237,16 +237,16 @@ const runtime_libs_lock = ReentrantLock() cached = Base.@lock runtime_libs_lock begin cached = get(runtime_libs, key, nothing) if cached === nothing || !runtime_library_valid(cached, job) - lib, sources, code_instances, host_references = build_runtime(job, config) + lib, sources, code_instances, relocations = build_runtime(job, config) io = IOBuffer() write(io, lib) cached = RuntimeLibrary(take!(io), sources, code_instances, job.world, - host_references) + relocations) runtime_libs[key] = cached end cached end return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true), - cached.host_references + cached.relocations end diff --git a/src/validation.jl b/src/validation.jl index 106e83c9..0e19d440 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -162,8 +162,8 @@ end # `show` via `showerror`, avoiding the default field-dump that derefs disposed IR Base.show(io::IO, err::InvalidIRError) = showerror(io, err) -function check_ir(job, mod::LLVM.Module, refs::HostReferences=HostReferences()) - errors = check_ir!(job, IRError[], mod, refs) +function check_ir(job, mod::LLVM.Module, relocs::Relocations=Relocations()) + errors = check_ir!(job, IRError[], mod, relocs) unique!(errors) if !isempty(errors) throw(InvalidIRError(job, errors)) @@ -172,9 +172,9 @@ function check_ir(job, mod::LLVM.Module, refs::HostReferences=HostReferences()) return end -function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module, refs::HostReferences) +function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module, relocs::Relocations) for f in functions(mod) - check_ir!(job, errors, f, refs) + check_ir!(job, errors, f, relocs) end # custom validation @@ -183,10 +183,10 @@ function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module, refs::HostRef return errors end -function check_ir!(job, errors::Vector{IRError}, f::LLVM.Function, refs::HostReferences) +function check_ir!(job, errors::Vector{IRError}, f::LLVM.Function, relocs::Relocations) for bb in blocks(f), inst in instructions(bb) if isa(inst, LLVM.CallInst) - check_ir!(job, errors, inst, refs) + check_ir!(job, errors, inst, relocs) elseif isa(inst, LLVM.LoadInst) check_ir!(job, errors, inst) end @@ -221,7 +221,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.LoadInst) return errors end -function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::HostReferences) +function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, relocs::Relocations) bt = backtrace(inst) dest = called_operand(inst) if isa(dest, LLVM.Function) @@ -233,7 +233,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::Host elseif fn == "jl_get_binding_or_error" || fn == "ijl_get_binding_or_error" try m, sym = arguments(inst) - ref = referenced_object(sym, refs) + ref = referenced_object(sym, relocs) ref === nothing && error("Unknown binding") push!(errors, (DELAYED_BINDING, bt, something(ref))) catch e @@ -244,7 +244,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::Host fn == "jl_get_binding_value_seqcst" || fn == "ijl_get_binding_value_seqcst" try # pry the binding from the IR - ref = referenced_object(arguments(inst)[1], refs) + ref = referenced_object(arguments(inst)[1], relocs) ref === nothing && error("Unknown binding") obj = something(ref) push!(errors, (DELAYED_BINDING, bt, obj.globalref)) @@ -255,7 +255,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::Host elseif startswith(fn, "tojlinvoke") try fun, args, nargs = arguments(inst) - ref = referenced_object(fun, refs) + ref = referenced_object(fun, relocs) ref === nothing && error("Unknown function") fun = something(ref)::Base.Function push!(errors, (DYNAMIC_CALL, bt, fun)) @@ -275,7 +275,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::Host end try fun, args, nargs, meth = arguments(inst) - ref = referenced_object(meth, refs) + ref = referenced_object(meth, relocs) ref === nothing && error("Unknown method instance") meth = something(ref)::Core.MethodInstance push!(errors, (DYNAMIC_CALL, bt, meth.def)) @@ -286,7 +286,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, refs::Host elseif fn == "jl_apply_generic" || fn == "ijl_apply_generic" try f, args, nargs = arguments(inst) - ref = referenced_object(f, refs) + ref = referenced_object(f, relocs) ref === nothing && error("Unknown function") f = something(ref) push!(errors, (DYNAMIC_CALL, bt, f)) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index e4035b5b..2a595cb9 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -24,14 +24,14 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint -function GPUCompiler.lower_host_references!(@nospecialize(job::NativeCompilerJob), +function GPUCompiler.lower_relocations!(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, - refs::GPUCompiler.HostReferences) + relocs::GPUCompiler.Relocations) if job.config.params.jit - GPUCompiler.emit_host_reference_declarations!(mod, refs) + GPUCompiler.emit_imported_relocations!(mod, relocs) else - invoke(GPUCompiler.lower_host_references!, - Tuple{CompilerJob,LLVM.Module,GPUCompiler.HostReferences}, job, mod, refs) + invoke(GPUCompiler.lower_relocations!, + Tuple{CompilerJob,LLVM.Module,GPUCompiler.Relocations}, job, mod, relocs) end end @@ -60,14 +60,14 @@ function create_job(@nospecialize(func), @nospecialize(types); CompilerJob(source, config), kwargs end -function load(obj::Vector{UInt8}, entry::String, refs::GPUCompiler.HostReferences) +function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations) lljit = LLJIT(; tm=JITTargetMachine()) try jd = JITDylib(lljit) prefix = LLVM.get_prefix(lljit) add!(jd, LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)) - relocations = GPUCompiler.resolved_relocations(refs) + relocations = GPUCompiler.resolved_relocations(relocs) cells = Vector{UInt}(undef, length(relocations.slots)) pairs = LLVM.API.LLVMOrcCSymbolMapPair[] for (i, (name, value)) in enumerate(relocations.slots) @@ -81,7 +81,7 @@ function load(obj::Vector{UInt8}, entry::String, refs::GPUCompiler.HostReference isempty(pairs) || LLVM.define(jd, LLVM.absolute_symbols(pairs)) add!(lljit, jd, MemoryBuffer(obj)) - for ((name, offset), value) in relocations.patches + for ((name, offset), value) in relocations.interior addr = lookup(lljit, name) unsafe_store!(Ptr{UInt}(pointer(addr) + offset), value) end diff --git a/test/native.jl b/test/native.jl index 03a953a6..8a2aec34 100644 --- a/test/native.jl +++ b/test/native.jl @@ -73,8 +73,8 @@ end @test only(other_mis).def in methods(mod.inner) if VERSION >= v"1.12" - @test length(meta.host_references.slots) == 1 - @test only(values(meta.host_references.slots)) isa GPUCompiler.JuliaValueRef + @test length(meta.relocations.slots) == 1 + @test only(values(meta.relocations.slots)) isa GPUCompiler.JuliaValueRef end end end @@ -250,7 +250,7 @@ end end end - @testset "runtime host references" begin + @testset "runtime relocations" begin # runtime functions like `box_bool` may reference Julia singletons through # `julia.constgv` globals. Keep their Julia identities with the cached bitcode # so the final kernel can resolve them in its own session. @@ -269,7 +269,7 @@ end used += 1 init = LLVM.initializer(gv) @test init === nothing - @test haskey(lib.host_references.slots, LLVM.name(gv)) + @test haskey(lib.relocations.slots, LLVM.name(gv)) end @static if VERSION >= v"1.12-" # on older versions, Julia bakes addresses without tagging globals @@ -347,7 +347,7 @@ end Native.code_execution(mod.egal_kernel, (Ptr{Bool}, Bool, Int32)) # Classification records whether the module stayed session-portable; eager - # lowering then resolves any remaining host-reference slots. + # lowering then resolves any remaining relocation slots. JuliaContext() do ctx # Unlike Int128, vector-shaped tuples are 16-byte aligned on all # supported architectures and Julia versions. @@ -370,10 +370,10 @@ end gv = LLVM.GlobalVariable(m, LLVM.PointerType(LLVM.Int8Type()), name) constant!(gv, true) end - refs = GPUCompiler.collect_julia_value_references!( + relocs = GPUCompiler.collect_julia_value_relocations!( m, Dict{String, Ptr{Cvoid}}()) - @test isempty(refs.patches) - GPUCompiler.resolve_host_references!(m, refs) + @test isempty(relocs.interior) + GPUCompiler.bake_relocations!(m, relocs) bool_ir = string(m) for name in ("jl_true", "jl_false") @test haskey(globals(m), "$(name)_box") @@ -386,17 +386,17 @@ end GC.@preserve objs begin # smalltag isbits: materialized, portable m, map = slot_module(ptrs[1]) - refs = GPUCompiler.collect_julia_value_references!(m, map) - @test isempty(refs.patches) - GPUCompiler.resolve_host_references!(m, refs) + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + @test isempty(relocs.interior) + GPUCompiler.bake_relocations!(m, relocs) @test haskey(globals(m), "jl_global_0_box") dispose(m) # Float64: the non-smalltag header is an interior relocation. m, map = slot_module(ptrs[2]) - refs = GPUCompiler.collect_julia_value_references!(m, map) - @test length(refs.patches) == 1 - (box_name, offset), ref = only(refs.patches) + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + @test length(relocs.interior) == 1 + (box_name, offset), ref = only(relocs.interior) @test offset == 0 @test ref.value === Float64 box = globals(m)[box_name] @@ -404,31 +404,31 @@ end @test linkage(box) == LLVM.API.LLVMExternalLinkage header_idx = Int(element_at(datalayout(m), global_value_type(box), offset)) + 1 @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == 0 - GPUCompiler.resolve_host_references!(m, refs) - @test isempty(refs.patches) + GPUCompiler.bake_relocations!(m, relocs) + @test isempty(relocs.interior) @test !isextinit(box) @test isconstant(box) @test linkage(box) == LLVM.API.LLVMPrivateLinkage @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == - GPUCompiler.resolve_host_reference(ref) + GPUCompiler.resolve_relocation_target(ref) dispose(m) # Symbol: baked address m, map = slot_module(ptrs[3]) - refs = GPUCompiler.collect_julia_value_references!(m, map) - @test only(values(refs.slots)).value === objs[3] - GPUCompiler.resolve_host_references!(m, refs) - @test isempty(refs.slots) + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + @test only(values(relocs.slots)).value === objs[3] + GPUCompiler.bake_relocations!(m, relocs) + @test isempty(relocs.slots) @test !haskey(globals(m), "jl_global_0_box") @test occursin("inttoptr", string(m)) dispose(m) # 16-byte-aligned payloads get padded past the header word m, map = slot_module(ptrs[4]) - refs = GPUCompiler.collect_julia_value_references!(m, map) - (box_name, offset), _ = only(refs.patches) + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + (box_name, offset), _ = only(relocs.interior) @test offset == 8 - GPUCompiler.resolve_host_references!(m, refs) + GPUCompiler.bake_relocations!(m, relocs) box = globals(m)[box_name] @test length(elements(LLVM.global_value_type(box))) == 3 dispose(m) @@ -735,13 +735,13 @@ end end end -@testset "host reference resolution" begin - sym = :host_reference_probe - @test GPUCompiler.resolve_host_reference(GPUCompiler.JuliaValueRef(sym)) == +@testset "relocation target resolution" begin + sym = :relocation_target_probe + @test GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(sym)) == UInt(pointer_from_objref(sym)) singleton = nothing - @test GPUCompiler.resolve_host_reference(GPUCompiler.JuliaValueRef(singleton)) == + @test GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(singleton)) == UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), singleton)) @test_throws ErrorException GPUCompiler.JuliaValueRef(1.5) @@ -750,20 +750,20 @@ end @testset "postponed relocation" begin if VERSION >= v"1.12" mod = @eval module $(gensym()) - f() = UInt(pointer_from_objref(:host_ref_probe)) + f() = UInt(pointer_from_objref(:relocation_probe)) end job, _ = Native.create_job(mod.f, Tuple{}; jit=true) JuliaContext() do ctx obj, meta = GPUCompiler.compile(:obj, job) - refs = meta.host_references - @test !isempty(refs.slots) - @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(refs.slots)) - @test all(name -> isdeclaration(globals(meta.ir)[name]), keys(refs.slots)) + relocs = meta.relocations + @test !isempty(relocs.slots) + @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(relocs.slots)) + @test all(name -> isdeclaration(globals(meta.ir)[name]), keys(relocs.slots)) bytes = Vector{UInt8}(codeunits(obj)) entry = LLVM.name(meta.entry) - expected = GPUCompiler.resolve_host_reference(only(values(refs.slots))) - fptr, keepalive = Native.load(bytes, entry, refs) + expected = GPUCompiler.resolve_relocation_target(only(values(relocs.slots))) + fptr, keepalive = Native.load(bytes, entry, relocs) try GC.@preserve keepalive begin actual = ccall(fptr, UInt, ()) @@ -773,7 +773,7 @@ end dispose(first(keepalive)) end - @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.HostReferences()) + @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.Relocations()) # Interior relocations are applied after the object has been loaded. ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" @@ -786,15 +786,15 @@ end }""") triple!(patch_mod, GPUCompiler.llvm_triple(job.config.target)) datalayout!(patch_mod, GPUCompiler.llvm_datalayout(job.config.target)) - patch_refs = GPUCompiler.HostReferences() - patch_refs.patches[("patched_box", 0)] = GPUCompiler.JuliaValueRef(Float64) + interior_relocs = GPUCompiler.Relocations() + interior_relocs.interior[("patched_box", 0)] = GPUCompiler.JuliaValueRef(Float64) patch_obj = GPUCompiler.mcgen(job, patch_mod, LLVM.API.LLVMObjectFile) patch_ptr, patch_keepalive = Native.load( - Vector{UInt8}(codeunits(patch_obj)), "patched_header", patch_refs) + Vector{UInt8}(codeunits(patch_obj)), "patched_header", interior_relocs) try GC.@preserve patch_keepalive begin @test ccall(patch_ptr, UInt, ()) == - GPUCompiler.resolve_host_reference( + GPUCompiler.resolve_relocation_target( GPUCompiler.JuliaValueRef(Float64)) end finally @@ -804,7 +804,7 @@ end end end -@testset "CPU reference resolution" begin +@testset "cglobal relocation" begin # JIT-private symbols like `jl_get_pgcstack_resolved` (JuliaLang/julia#61527) cannot # be looked up using `jl_cglobal`, so we should only resolve bindings that are # actually loaded from, leaving called functions alone. @@ -844,12 +844,12 @@ end %value = load $word_ptr, $word_ptr_ptr @jl_float32_type ret $word_ptr %value }""") - refs = GPUCompiler.HostReferences() - @test GPUCompiler.collect_runtime_global_references!(mod, refs) - name = only(keys(refs.slots)) - @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + relocs = GPUCompiler.Relocations() + @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) + name = only(keys(relocs.slots)) + @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) - GPUCompiler.emit_host_reference_definitions!(mod, refs) + GPUCompiler.emit_patchable_relocations!(mod, relocs) @test occursin("externally_initialized global i64 0", string(mod)) mod = parse(LLVM.Module, """ @@ -859,13 +859,13 @@ end %value = load i64, $(function_word_ptr("jl_float32_type")) ret i64 %value }""") - refs = GPUCompiler.HostReferences() - @test GPUCompiler.collect_runtime_global_references!(mod, refs) - name = only(keys(refs.slots)) - @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + relocs = GPUCompiler.Relocations() + @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) + name = only(keys(relocs.slots)) + @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$name = external global i64", string(mod)) - GPUCompiler.emit_host_reference_declarations!(mod, refs) - @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + GPUCompiler.emit_imported_relocations!(mod, relocs) + @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test isdeclaration(globals(mod)[name]) @test occursin("@$name = external global i64", string(mod)) @@ -876,17 +876,17 @@ end %value = load $word_ptr, $word_ptr_ptr @jl_float32_type ret $word_ptr %value }""") - refs = GPUCompiler.HostReferences() - @test GPUCompiler.collect_runtime_global_references!(mod, refs) - name = only(keys(refs.slots)) - GPUCompiler.emit_host_reference_declarations!(mod, refs) - @test refs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) + relocs = GPUCompiler.Relocations() + @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) + name = only(keys(relocs.slots)) + GPUCompiler.emit_imported_relocations!(mod, relocs) + @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) @test isdeclaration(globals(mod)[name]) @test occursin("@$name = external global i64", string(mod)) end end -@testset "host reference linking" begin +@testset "relocation linking" begin JuliaContext() do ctx ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" @@ -900,26 +900,26 @@ end }""") end - refs(name, value) = - GPUCompiler.HostReferences(Dict(name => GPUCompiler.JuliaValueRef(value)), - Dict{Tuple{String,Int},GPUCompiler.HostReference}()) + slot_relocs(name, value) = + GPUCompiler.Relocations(Dict(name => GPUCompiler.JuliaValueRef(value)), + Dict{Tuple{String,Int},GPUCompiler.RelocationTarget}()) # Equal Julia identities deliberately share a single slot. dest = slot_module("slot", "first") - dest_refs = refs("slot", :shared) + dest_relocs = slot_relocs("slot", :shared) src = slot_module("slot", "second") - src_refs = refs("slot", :shared) - GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) - @test collect(keys(dest_refs.slots)) == ["slot"] + src_relocs = slot_relocs("slot", :shared) + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs) + @test collect(keys(dest_relocs.slots)) == ["slot"] @test occursin("@slot = external global i64", string(dest)) # The name is the slot identity, so conflicting metadata is an error. dest = slot_module("slot", "first") - dest_refs = refs("slot", :first) + dest_relocs = slot_relocs("slot", :first) src = slot_module("slot", "second") - src_refs = refs("slot", :second) - @test_throws ErrorException GPUCompiler.link_with_host_references!( - dest, dest_refs, src, src_refs) + src_relocs = slot_relocs("slot", :second) + @test_throws ErrorException GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) # `only_needed` must keep metadata for imported slots and discard metadata for # source globals that the LLVM linker did not import. @@ -938,17 +938,17 @@ end %value = load i64, $(ptr("i64")) @used ret i64 %value }""") - src_refs = GPUCompiler.HostReferences(Dict( + src_relocs = GPUCompiler.Relocations(Dict( "used" => GPUCompiler.JuliaValueRef(:used), "unused" => GPUCompiler.JuliaValueRef(:unused), - ), Dict{Tuple{String,Int},GPUCompiler.HostReference}()) - dest_refs = GPUCompiler.HostReferences() - GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs; - only_needed=true) - @test collect(keys(dest_refs.slots)) == ["used"] - @test only(values(dest_refs.slots)).value === :used - - function patch_module(name, entry) + ), Dict{Tuple{String,Int},GPUCompiler.RelocationTarget}()) + dest_relocs = GPUCompiler.Relocations() + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; + only_needed=true) + @test collect(keys(dest_relocs.slots)) == ["used"] + @test only(values(dest_relocs.slots)).value === :used + + function interior_module(name, entry) parse(LLVM.Module, """ @$name = externally_initialized global { i64, i64 } { i64 0, i64 1 } @@ -957,44 +957,44 @@ end ret i64 %value }""") end - patch_refs(name, value) = GPUCompiler.HostReferences( - Dict{String,GPUCompiler.HostReference}(), + interior_relocs(name, value) = GPUCompiler.Relocations( + Dict{String,GPUCompiler.RelocationTarget}(), Dict((name, 0) => GPUCompiler.JuliaValueRef(value))) - # Identical patch identities merge; conflicting metadata does not. - dest = patch_module("patch", "first_patch") - dest_refs = patch_refs("patch", Float64) - src = patch_module("patch", "second_patch") - src_refs = patch_refs("patch", Float64) - GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs) - @test collect(keys(dest_refs.patches)) == [("patch", 0)] - - dest = patch_module("patch", "first_patch") - dest_refs = patch_refs("patch", Float64) - src = patch_module("patch", "second_patch") - src_refs = patch_refs("patch", Int64) - @test_throws ErrorException GPUCompiler.link_with_host_references!( - dest, dest_refs, src, src_refs) - - # Metadata for patch globals not imported under `only_needed` is discarded. + # Identical interior relocation identities merge; conflicting metadata does not. + dest = interior_module("patch", "first_patch") + dest_relocs = interior_relocs("patch", Float64) + src = interior_module("patch", "second_patch") + src_relocs = interior_relocs("patch", Float64) + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs) + @test collect(keys(dest_relocs.interior)) == [("patch", 0)] + + dest = interior_module("patch", "first_patch") + dest_relocs = interior_relocs("patch", Float64) + src = interior_module("patch", "second_patch") + src_relocs = interior_relocs("patch", Int64) + @test_throws ErrorException GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) + + # Metadata for interior globals not imported under `only_needed` is discarded. dest = parse(LLVM.Module, """ declare i64 @source_patch() define i64 @entry_patch() { %value = call i64 @source_patch() ret i64 %value }""") - src = patch_module("used_patch", "source_patch") + src = interior_module("used_patch", "source_patch") unused = GlobalVariable(src, LLVM.StructType([LLVM.Int64Type(), LLVM.Int64Type()]), "unused_patch") initializer!(unused, ConstantStruct(LLVM.Constant[ConstantInt(0), ConstantInt(1)])) - src_refs = GPUCompiler.HostReferences( - Dict{String,GPUCompiler.HostReference}(), + src_relocs = GPUCompiler.Relocations( + Dict{String,GPUCompiler.RelocationTarget}(), Dict(("used_patch", 0) => GPUCompiler.JuliaValueRef(Float64), ("unused_patch", 0) => GPUCompiler.JuliaValueRef(Int64))) - dest_refs = GPUCompiler.HostReferences() - GPUCompiler.link_with_host_references!(dest, dest_refs, src, src_refs; - only_needed=true) - @test collect(keys(dest_refs.patches)) == [("used_patch", 0)] + dest_relocs = GPUCompiler.Relocations() + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; + only_needed=true) + @test collect(keys(dest_relocs.interior)) == [("used_patch", 0)] end end diff --git a/test/ptx.jl b/test/ptx.jl index 56099aa2..4f44c0af 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -30,7 +30,7 @@ end @testset "global variable relocation" begin # references to Julia objects (`julia.constgv` globals, e.g. Symbol literals) must - # survive as host-reference slots until backend lowering at object emission. + # survive as relocation slots until backend lowering at object emission. # they used to be kept alive as internal globals with a null initializer, which the # GlobalOpt run in `finish_module!` folded away, constant-folding any comparison # against them (JuliaGPU/CUDA.jl#3185: kernels specialized on Symbols misbehaved). From 09bd2c3d122ebe3e8f1712a6db937ef28d2984b1 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Jul 2026 08:51:37 +0200 Subject: [PATCH 28/45] Unify relocation site metadata --- src/interface.jl | 2 +- src/relocation.jl | 253 ++++++++++++++++++++--------------------- test/helpers/native.jl | 18 +-- test/native.jl | 116 ++++++++++--------- 4 files changed, 198 insertions(+), 191 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index a715a207..117e3b62 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -512,7 +512,7 @@ end end # HAS_INTEGRATED_CACHE -@public JuliaValueRef, CGlobalRef, RelocationTarget, Relocations +@public JuliaValueRef, CGlobalRef, RelocationTarget, RelocationSite, Relocations @public lower_relocations!, bake_relocations! @public emit_patchable_relocations!, emit_imported_relocations! @public resolve_relocation_target, resolved_relocations, supports_relocatable_ir diff --git a/src/relocation.jl b/src/relocation.jl index 5a68221b..9d44334f 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -2,18 +2,19 @@ # # A relocation is a site => target pair. Targets are Julia object identities # (`JuliaValueRef`, like codegen's identity-keyed `global_targets`) or named libjulia -# globals (`CGlobalRef`, like JuliaVariables). Sites are either dedicated word-sized -# `slots` (like `jl_sysimg_gvars`) or word-sized `interior` locations inside defined -# globals (like `gctags_list`/`relocs_list`). Slots can be baked, patched, or imported; -# interior sites can only be baked or patched. +# globals (`CGlobalRef`, like JuliaVariables). Every site names a word at a byte offset in a +# global. Dedicated sites use offset zero in a word-sized declaration (like +# `jl_sysimg_gvars`); other sites are inside defined globals (like +# `gctags_list`/`relocs_list`). Declarations can be baked, patched, or imported; sites in +# definitions can only be baked or patched. # # produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) # # Lowering strategy Loader contract Julia analog # `bake_relocations!` none; current-session words baked JIT address folding # `emit_patchable_relocations!` patch named globals after load sysimage gvars -# `emit_imported_relocations!` import slots before load; patch ORC absoluteSymbols -# interior sites afterward +# `emit_imported_relocations!` import declarations before load; ORC absoluteSymbols +# patch definitions afterward # # The producers are `collect_julia_value_relocations!` during IR generation and # `collect_cglobal_relocations!` immediately before backend lowering. Names are globally @@ -83,39 +84,44 @@ end ## The table """ - Relocations(slots, interior) + RelocationSite(name, offset) -Relocation metadata accompanying a module. `slots` maps dedicated word-sized global names -to targets. `interior` maps a defined global name and byte offset to targets. See -[`resolved_relocations`](@ref) for resolving both site kinds for a loader. +The location of a word to relocate: a global name and a byte offset within that global. +The global's IR shape determines how the site is lowered. A declaration denotes an +importable word-sized slot and must have offset zero; a definition denotes a post-load +patch within its initializer. +""" +struct RelocationSite + name::String + offset::Int +end + +""" + Relocations(sites) + +Relocation metadata accompanying a module. `sites` maps [`RelocationSite`](@ref)s to +targets. See [`resolved_relocations`](@ref) for resolving sites for a loader. """ struct Relocations - slots::Dict{String,RelocationTarget} - interior::Dict{Tuple{String,Int},RelocationTarget} + sites::Dict{RelocationSite,RelocationTarget} end -Relocations() = Relocations(Dict{String,RelocationTarget}(), - Dict{Tuple{String,Int},RelocationTarget}()) +Relocations() = Relocations(Dict{RelocationSite,RelocationTarget}()) """ resolved_relocations(relocs) -Resolve relocation metadata for a loader. Returns resolved `slots`, resolved `interior`, and -Julia `roots` that must stay alive while the loaded code can access them. +Resolve relocation metadata for a loader. Returns resolved `sites` and Julia `roots` that +must stay alive while the loaded code can access them. """ function resolved_relocations(relocs::Relocations) - slots = Pair{String,UInt}[] - interior = Pair{Tuple{String,Int},UInt}[] + sites = Pair{RelocationSite,UInt}[] roots = Any[] - for (name, ref) in relocs.slots - push!(slots, name => resolve_relocation_target(ref)) - ref isa JuliaValueRef && push!(roots, ref.value) - end - for (key, ref) in relocs.interior - push!(interior, key => resolve_relocation_target(ref)) + for (site, ref) in relocs.sites + push!(sites, site => resolve_relocation_target(ref)) ref isa JuliaValueRef && push!(roots, ref.value) end - return (; slots, interior, roots) + return (; sites, roots) end relocation_word_type() = LLVM.IntType(8sizeof(UInt)) @@ -137,31 +143,28 @@ function slot_initializer(gv::GlobalVariable, value::UInt) error("Relocation slot '$(LLVM.name(gv))' has unsupported LLVM type $T") end -function foreach_slot_relocation(f, mod::LLVM.Module, relocs::Relocations) +function foreach_relocation(f, mod::LLVM.Module, relocs::Relocations) mod_gvs = globals(mod) - for (name, ref) in relocs.slots - haskey(mod_gvs, name) || error("Missing relocation slot '$name'") + for (site, ref) in relocs.sites + name = site.name + offset = site.offset + haskey(mod_gvs, name) || error("Missing relocation global '$name'") gv = mod_gvs[name] - check_slot_size(mod, gv, name) - f(name, gv, ref) - end - return -end - -function foreach_interior_relocation(f, mod::LLVM.Module, relocs::Relocations) - mod_gvs = globals(mod) - for ((name, offset), ref) in relocs.interior - haskey(mod_gvs, name) || error("Missing interior relocation global '$name'") - gv = mod_gvs[name] - init = initializer(gv) - init === nothing && error("Interior relocation global '$name' has no initializer") - T = value_type(init) - T isa LLVM.StructType || - error("Interior relocation global '$name' has non-struct initializer $T") - size = abi_size(datalayout(mod), T) - 0 <= offset && offset + sizeof(UInt) <= size || - error("Interior relocation '$name+$offset' is outside its $size-byte global") - f(name, offset, gv, ref) + if isdeclaration(gv) + offset == 0 || + error("Relocation declaration '$name' has nonzero offset $offset") + check_slot_size(mod, gv, name) + else + init = initializer(gv) + init === nothing && error("Relocation global '$name' has no initializer") + T = value_type(init) + T isa LLVM.StructType || + error("Relocation global '$name' has non-struct initializer $T") + size = abi_size(datalayout(mod), T) + 0 <= offset && offset + sizeof(UInt) <= size || + error("Relocation '$name+$offset' is outside its $size-byte global") + end + f(site, gv, ref) end return end @@ -195,7 +198,7 @@ function collect_julia_value_relocations!(mod::LLVM.Module, LLVM.name!(gv, slot_name) LLVM.name(gv) == slot_name || error("Relocation slot name '$slot_name' is already in use") - relocs.slots[slot_name] = JuliaValueRef(obj) + relocs.sites[RelocationSite(slot_name, 0)] = JuliaValueRef(obj) end end @@ -263,7 +266,7 @@ function materialize_box!(mod::LLVM.Module, relocs::Relocations, gv::GlobalVaria linkage!(box, LLVM.API.LLVMExternalLinkage) extinit!(box, true) offset = Int(offsetof(datalayout(mod), boxty, header_idx)) - relocs.interior[(box_name, offset)] = JuliaValueRef(typeof(obj)) + relocs.sites[RelocationSite(box_name, offset)] = JuliaValueRef(typeof(obj)) else constant!(box, true) linkage!(box, LLVM.API.LLVMPrivateLinkage) @@ -278,10 +281,11 @@ function materialize_box!(mod::LLVM.Module, relocs::Relocations, gv::GlobalVaria end # Some Julia code loads words from libjulia C globals, for example type tags. Record those -# loads as slot relocations immediately before object emission. +# loads as dedicated zero-offset relocations immediately before object emission. function is_cglobal_candidate(value, relocs::Relocations) name = LLVM.name(value) - value isa LLVM.GlobalVariable && haskey(relocs.slots, name) && return false + value isa LLVM.GlobalVariable && + haskey(relocs.sites, RelocationSite(name, 0)) && return false isdeclaration(value) || return false value isa LLVM.Function && LLVM.isintrinsic(value) && return false return startswith(name, "jl_") @@ -300,7 +304,7 @@ function collect_cglobal_relocations!(mod::LLVM.Module, relocs::Relocations) slot = GlobalVariable(mod, relocation_word_type(), name) LLVM.name(slot) == name || error("cglobal slot name '$name' is already in use") - relocs.slots[name] = CGlobalRef(Symbol(fn)) + relocs.sites[RelocationSite(name, 0)] = CGlobalRef(Symbol(fn)) end slot end @@ -362,55 +366,51 @@ end function link_relocatable!(dest_mod::LLVM.Module, dest_relocs::Relocations, src_mod::LLVM.Module, src_relocs::Relocations; only_needed=false) - # Interior relocation globals are definitions, unlike slots. Make an identical source - # definition a declaration so LLVM can resolve it to the destination while linking. - for (key, ref) in src_relocs.interior - existing = get(dest_relocs.interior, key, nothing) + # Make an identical source definition a declaration so LLVM can resolve it to the + # destination while linking. Declaration sites already merge without modification. + for (site, ref) in src_relocs.sites + existing = get(dest_relocs.sites, site, nothing) existing === nothing && continue - name, offset = key + name = site.name + offset = site.offset same_relocation_target(existing, ref) || - error("Interior relocation '$name+$offset' refers to conflicting values") - haskey(globals(dest_mod), name) || - error("Missing destination interior relocation global '$name'") + error("Relocation '$name+$offset' refers to conflicting values") haskey(globals(src_mod), name) || - error("Missing source interior relocation global '$name'") + error("Missing source relocation global '$name'") gv = globals(src_mod)[name] + isdeclaration(gv) && continue + haskey(globals(dest_mod), name) || + error("Missing destination relocation global '$name'") initializer!(gv, nothing) extinit!(gv, false) linkage!(gv, LLVM.API.LLVMExternalLinkage) end link!(dest_mod, src_mod; only_needed) - for (name, ref) in src_relocs.slots - # A slot absent from the linked module was dead (DCE'd or not imported under + for (site, ref) in src_relocs.sites + name = site.name + # A site absent from the linked module was dead (DCE'd or not imported under # `only_needed`); its relocation dies with it. haskey(globals(dest_mod), name) || continue - existing = get(dest_relocs.slots, name, nothing) + existing = get(dest_relocs.sites, site, nothing) existing === nothing || same_relocation_target(existing, ref) || - error("Relocation slot '$name' refers to conflicting values") - dest_relocs.slots[name] = ref - end - for ((name, offset), ref) in src_relocs.interior - haskey(globals(dest_mod), name) || continue - key = (name, offset) - existing = get(dest_relocs.interior, key, nothing) - existing === nothing || same_relocation_target(existing, ref) || - error("Interior relocation '$name+$offset' refers to conflicting values") - dest_relocs.interior[key] = ref + error("Relocation '$(site.name)+$(site.offset)' refers to conflicting values") + dest_relocs.sites[site] = ref end return end function prune_dead_relocations!(mod::LLVM.Module, relocs::Relocations) mod_gvs = globals(mod) - for name in collect(keys(relocs.slots)) - haskey(mod_gvs, name) || delete!(relocs.slots, name) - end - for ((name, offset), _) in collect(relocs.interior) + for name in unique(site.name for site in keys(relocs.sites)) if !haskey(mod_gvs, name) - delete!(relocs.interior, (name, offset)) - elseif isempty(uses(mod_gvs[name])) - delete!(relocs.interior, (name, offset)) + for site in collect(keys(relocs.sites)) + site.name == name && delete!(relocs.sites, site) + end + elseif !isdeclaration(mod_gvs[name]) && isempty(uses(mod_gvs[name])) + for site in collect(keys(relocs.sites)) + site.name == name && delete!(relocs.sites, site) + end erase!(mod_gvs[name]) end end @@ -429,7 +429,7 @@ Backends may instead emit patchable globals or imported slot symbols for their l Backends using baked lowering must not persist generated code in `cached_results`. Generated code is session-portable only when [`supports_relocatable_ir`](@ref) and the backend preserves -all slot and interior relocations for its loader. +all relocations for its loader. """ function lower_relocations!(@nospecialize(job::CompilerJob), mod::LLVM.Module, relocs::Relocations) @@ -437,46 +437,45 @@ function lower_relocations!(@nospecialize(job::CompilerJob), mod::LLVM.Module, end function bake_relocations!(mod::LLVM.Module, relocs::Relocations) - foreach_slot_relocation(mod, relocs) do name, gv, ref - value = resolve_relocation_target(ref) - val = slot_initializer(gv, value) - initializer!(gv, val) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - constant!(gv, true) - end - foreach_interior_relocation(mod, relocs) do name, offset, gv, ref - init = initializer(gv) - T = value_type(init)::LLVM.StructType - idx = Int(element_at(datalayout(mod), T, offset)) + 1 - fields = LLVM.Constant[operands(init)...] - fields[idx] = ConstantInt(value_type(fields[idx]), resolve_relocation_target(ref)) - initializer!(gv, ConstantStruct(T, fields)) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - extinit!(gv, false) - constant!(gv, true) - unnamed_addr!(gv, true) + foreach_relocation(mod, relocs) do site, gv, ref + if isdeclaration(gv) + value = resolve_relocation_target(ref) + val = slot_initializer(gv, value) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + constant!(gv, true) + else + init = initializer(gv) + T = value_type(init)::LLVM.StructType + idx = Int(element_at(datalayout(mod), T, site.offset)) + 1 + fields = LLVM.Constant[operands(init)...] + fields[idx] = ConstantInt(value_type(fields[idx]), resolve_relocation_target(ref)) + initializer!(gv, ConstantStruct(T, fields)) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + extinit!(gv, false) + constant!(gv, true) + unnamed_addr!(gv, true) + end end - empty!(relocs.slots) - empty!(relocs.interior) + empty!(relocs.sites) return end """ emit_patchable_relocations!(mod, relocs) -Emit slots as writable, null-initialized definitions. The loader must patch each definition -and interior relocation after loading the object. This requires a per-object symbol namespace. +Emit declarations as writable, null-initialized definitions. The loader must patch every +relocation after loading the object. This requires a per-object symbol namespace. """ function emit_patchable_relocations!(mod::LLVM.Module, relocs::Relocations) used = GlobalVariable[] - foreach_slot_relocation(mod, relocs) do _, gv, _ - initializer!(gv, null(global_value_type(gv))) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, true) - push!(used, gv) - end - foreach_interior_relocation(mod, relocs) do _, _, gv, _ + foreach_relocation(mod, relocs) do _, gv, _ + if isdeclaration(gv) + initializer!(gv, null(global_value_type(gv))) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, true) + end push!(used, gv) end isempty(used) || set_used!(mod, used...) @@ -486,23 +485,23 @@ end """ emit_imported_relocations!(mod, relocs) -Prepare relocations for a loader that imports slot symbols before loading an object. +Prepare relocations for a loader that imports declaration symbols before loading an object. -Every slot remains an external word-sized declaration. The loader must define each symbol to -point at a cell containing [`resolve_relocation_target`](@ref), and keep the cell and any -referenced Julia value alive while the code is executable. Interior sites cannot be imported -before load and must always be written afterward. +Every declaration remains an external word-sized declaration. The loader must define its +symbol to point at a cell containing [`resolve_relocation_target`](@ref), and keep the cell +and any referenced Julia value alive while the code is executable. Sites in definitions +cannot be imported before load and must always be written afterward. """ function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) used = GlobalVariable[] - foreach_slot_relocation(mod, relocs) do name, gv, _ - isdeclaration(gv) || error("Relocation slot '$name' must be a declaration") - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, false) - end - foreach_interior_relocation(mod, relocs) do _, _, gv, _ - push!(used, gv) + foreach_relocation(mod, relocs) do site, gv, _ + if isdeclaration(gv) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + else + push!(used, gv) + end end isempty(used) || set_used!(mod, used...) return @@ -523,7 +522,7 @@ function referenced_object(value, relocs::Relocations) source = first(operands(source)) end if source isa GlobalVariable - ref = get(relocs.slots, LLVM.name(source), nothing) + ref = get(relocs.sites, RelocationSite(LLVM.name(source), 0), nothing) ref isa JuliaValueRef && return Some(ref.value) end elseif value isa ConstantExpr && opcode(value) == LLVM.API.LLVMIntToPtr diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 2a595cb9..08f9fa52 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -60,7 +60,8 @@ function create_job(@nospecialize(func), @nospecialize(types); CompilerJob(source, config), kwargs end -function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations) +function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations, + ir::LLVM.Module) lljit = LLJIT(; tm=JITTargetMachine()) try jd = JITDylib(lljit) @@ -68,22 +69,25 @@ function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations add!(jd, LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)) relocations = GPUCompiler.resolved_relocations(relocs) - cells = Vector{UInt}(undef, length(relocations.slots)) + declarations = [(site, value) for (site, value) in relocations.sites + if isdeclaration(globals(ir)[site.name])] + cells = Vector{UInt}(undef, length(declarations)) pairs = LLVM.API.LLVMOrcCSymbolMapPair[] - for (i, (name, value)) in enumerate(relocations.slots) + for (i, (site, value)) in enumerate(declarations) cells[i] = value symbol = LLVM.API.LLVMJITEvaluatedSymbol( reinterpret(UInt, pointer(cells, i)), LLVM.API.LLVMJITSymbolFlags( LLVM.API.LLVMJITSymbolGenericFlagsExported, 0)) - push!(pairs, LLVM.API.LLVMOrcCSymbolMapPair(mangle(lljit, name), symbol)) + push!(pairs, LLVM.API.LLVMOrcCSymbolMapPair(mangle(lljit, site.name), symbol)) end isempty(pairs) || LLVM.define(jd, LLVM.absolute_symbols(pairs)) add!(lljit, jd, MemoryBuffer(obj)) - for ((name, offset), value) in relocations.interior - addr = lookup(lljit, name) - unsafe_store!(Ptr{UInt}(pointer(addr) + offset), value) + for (site, value) in relocations.sites + isdeclaration(globals(ir)[site.name]) && continue + addr = lookup(lljit, site.name) + unsafe_store!(Ptr{UInt}(pointer(addr) + site.offset), value) end addr = lookup(lljit, entry) return pointer(addr), (lljit, cells, relocations.roots) diff --git a/test/native.jl b/test/native.jl index 8a2aec34..04957dab 100644 --- a/test/native.jl +++ b/test/native.jl @@ -73,8 +73,8 @@ end @test only(other_mis).def in methods(mod.inner) if VERSION >= v"1.12" - @test length(meta.relocations.slots) == 1 - @test only(values(meta.relocations.slots)) isa GPUCompiler.JuliaValueRef + @test length(meta.relocations.sites) == 1 + @test only(values(meta.relocations.sites)) isa GPUCompiler.JuliaValueRef end end end @@ -269,7 +269,8 @@ end used += 1 init = LLVM.initializer(gv) @test init === nothing - @test haskey(lib.relocations.slots, LLVM.name(gv)) + site = GPUCompiler.RelocationSite(LLVM.name(gv), 0) + @test haskey(lib.relocations.sites, site) end @static if VERSION >= v"1.12-" # on older versions, Julia bakes addresses without tagging globals @@ -372,7 +373,7 @@ end end relocs = GPUCompiler.collect_julia_value_relocations!( m, Dict{String, Ptr{Cvoid}}()) - @test isempty(relocs.interior) + @test isempty(relocs.sites) GPUCompiler.bake_relocations!(m, relocs) bool_ir = string(m) for name in ("jl_true", "jl_false") @@ -387,7 +388,7 @@ end # smalltag isbits: materialized, portable m, map = slot_module(ptrs[1]) relocs = GPUCompiler.collect_julia_value_relocations!(m, map) - @test isempty(relocs.interior) + @test isempty(relocs.sites) GPUCompiler.bake_relocations!(m, relocs) @test haskey(globals(m), "jl_global_0_box") dispose(m) @@ -395,17 +396,18 @@ end # Float64: the non-smalltag header is an interior relocation. m, map = slot_module(ptrs[2]) relocs = GPUCompiler.collect_julia_value_relocations!(m, map) - @test length(relocs.interior) == 1 - (box_name, offset), ref = only(relocs.interior) - @test offset == 0 + @test length(relocs.sites) == 1 + site, ref = only(relocs.sites) + @test site.offset == 0 @test ref.value === Float64 - box = globals(m)[box_name] + box = globals(m)[site.name] @test isextinit(box) @test linkage(box) == LLVM.API.LLVMExternalLinkage - header_idx = Int(element_at(datalayout(m), global_value_type(box), offset)) + 1 + header_idx = Int(element_at(datalayout(m), global_value_type(box), + site.offset)) + 1 @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == 0 GPUCompiler.bake_relocations!(m, relocs) - @test isempty(relocs.interior) + @test isempty(relocs.sites) @test !isextinit(box) @test isconstant(box) @test linkage(box) == LLVM.API.LLVMPrivateLinkage @@ -416,9 +418,9 @@ end # Symbol: baked address m, map = slot_module(ptrs[3]) relocs = GPUCompiler.collect_julia_value_relocations!(m, map) - @test only(values(relocs.slots)).value === objs[3] + @test only(values(relocs.sites)).value === objs[3] GPUCompiler.bake_relocations!(m, relocs) - @test isempty(relocs.slots) + @test isempty(relocs.sites) @test !haskey(globals(m), "jl_global_0_box") @test occursin("inttoptr", string(m)) dispose(m) @@ -426,10 +428,10 @@ end # 16-byte-aligned payloads get padded past the header word m, map = slot_module(ptrs[4]) relocs = GPUCompiler.collect_julia_value_relocations!(m, map) - (box_name, offset), _ = only(relocs.interior) - @test offset == 8 + site, _ = only(relocs.sites) + @test site.offset == 8 GPUCompiler.bake_relocations!(m, relocs) - box = globals(m)[box_name] + box = globals(m)[site.name] @test length(elements(LLVM.global_value_type(box))) == 3 dispose(m) end @@ -756,14 +758,14 @@ end JuliaContext() do ctx obj, meta = GPUCompiler.compile(:obj, job) relocs = meta.relocations - @test !isempty(relocs.slots) - @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(relocs.slots)) - @test all(name -> isdeclaration(globals(meta.ir)[name]), keys(relocs.slots)) + @test !isempty(relocs.sites) + @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(relocs.sites)) + @test all(site -> isdeclaration(globals(meta.ir)[site.name]), keys(relocs.sites)) bytes = Vector{UInt8}(codeunits(obj)) entry = LLVM.name(meta.entry) - expected = GPUCompiler.resolve_relocation_target(only(values(relocs.slots))) - fptr, keepalive = Native.load(bytes, entry, relocs) + expected = GPUCompiler.resolve_relocation_target(only(values(relocs.sites))) + fptr, keepalive = Native.load(bytes, entry, relocs, meta.ir) try GC.@preserve keepalive begin actual = ccall(fptr, UInt, ()) @@ -773,9 +775,10 @@ end dispose(first(keepalive)) end - @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.Relocations()) + @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.Relocations(), + meta.ir) - # Interior relocations are applied after the object has been loaded. + # Sites in definitions are applied after the object has been loaded. ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" patch_mod = parse(LLVM.Module, """ @patched_box = externally_initialized global { i64, i64 } { i64 0, i64 42 } @@ -786,11 +789,13 @@ end }""") triple!(patch_mod, GPUCompiler.llvm_triple(job.config.target)) datalayout!(patch_mod, GPUCompiler.llvm_datalayout(job.config.target)) - interior_relocs = GPUCompiler.Relocations() - interior_relocs.interior[("patched_box", 0)] = GPUCompiler.JuliaValueRef(Float64) + patch_relocs = GPUCompiler.Relocations() + site = GPUCompiler.RelocationSite("patched_box", 0) + patch_relocs.sites[site] = GPUCompiler.JuliaValueRef(Float64) patch_obj = GPUCompiler.mcgen(job, patch_mod, LLVM.API.LLVMObjectFile) patch_ptr, patch_keepalive = Native.load( - Vector{UInt8}(codeunits(patch_obj)), "patched_header", interior_relocs) + Vector{UInt8}(codeunits(patch_obj)), "patched_header", patch_relocs, + patch_mod) try GC.@preserve patch_keepalive begin @test ccall(patch_ptr, UInt, ()) == @@ -846,9 +851,10 @@ end }""") relocs = GPUCompiler.Relocations() @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) - name = only(keys(relocs.slots)) - @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) - @test occursin("@$name = external global i64", string(mod)) + site = only(keys(relocs.sites)) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test site.offset == 0 + @test occursin("@$(site.name) = external global i64", string(mod)) GPUCompiler.emit_patchable_relocations!(mod, relocs) @test occursin("externally_initialized global i64 0", string(mod)) @@ -861,13 +867,13 @@ end }""") relocs = GPUCompiler.Relocations() @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) - name = only(keys(relocs.slots)) - @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) - @test occursin("@$name = external global i64", string(mod)) + site = only(keys(relocs.sites)) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test occursin("@$(site.name) = external global i64", string(mod)) GPUCompiler.emit_imported_relocations!(mod, relocs) - @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) - @test isdeclaration(globals(mod)[name]) - @test occursin("@$name = external global i64", string(mod)) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test isdeclaration(globals(mod)[site.name]) + @test occursin("@$(site.name) = external global i64", string(mod)) mod = parse(LLVM.Module, """ @jl_float32_type = external global $word_ptr @@ -878,11 +884,11 @@ end }""") relocs = GPUCompiler.Relocations() @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) - name = only(keys(relocs.slots)) + site = only(keys(relocs.sites)) GPUCompiler.emit_imported_relocations!(mod, relocs) - @test relocs.slots[name] == GPUCompiler.CGlobalRef(:jl_float32_type) - @test isdeclaration(globals(mod)[name]) - @test occursin("@$name = external global i64", string(mod)) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test isdeclaration(globals(mod)[site.name]) + @test occursin("@$(site.name) = external global i64", string(mod)) end end @@ -900,9 +906,9 @@ end }""") end - slot_relocs(name, value) = - GPUCompiler.Relocations(Dict(name => GPUCompiler.JuliaValueRef(value)), - Dict{Tuple{String,Int},GPUCompiler.RelocationTarget}()) + site(name, offset=0) = GPUCompiler.RelocationSite(name, offset) + slot_relocs(name, value) = GPUCompiler.Relocations( + Dict(site(name) => GPUCompiler.JuliaValueRef(value))) # Equal Julia identities deliberately share a single slot. dest = slot_module("slot", "first") @@ -910,7 +916,7 @@ end src = slot_module("slot", "second") src_relocs = slot_relocs("slot", :shared) GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs) - @test collect(keys(dest_relocs.slots)) == ["slot"] + @test collect(keys(dest_relocs.sites)) == [site("slot")] @test occursin("@slot = external global i64", string(dest)) # The name is the slot identity, so conflicting metadata is an error. @@ -939,14 +945,14 @@ end ret i64 %value }""") src_relocs = GPUCompiler.Relocations(Dict( - "used" => GPUCompiler.JuliaValueRef(:used), - "unused" => GPUCompiler.JuliaValueRef(:unused), - ), Dict{Tuple{String,Int},GPUCompiler.RelocationTarget}()) + site("used") => GPUCompiler.JuliaValueRef(:used), + site("unused") => GPUCompiler.JuliaValueRef(:unused), + )) dest_relocs = GPUCompiler.Relocations() GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; only_needed=true) - @test collect(keys(dest_relocs.slots)) == ["used"] - @test only(values(dest_relocs.slots)).value === :used + @test collect(keys(dest_relocs.sites)) == [site("used")] + @test only(values(dest_relocs.sites)).value === :used function interior_module(name, entry) parse(LLVM.Module, """ @@ -958,8 +964,7 @@ end }""") end interior_relocs(name, value) = GPUCompiler.Relocations( - Dict{String,GPUCompiler.RelocationTarget}(), - Dict((name, 0) => GPUCompiler.JuliaValueRef(value))) + Dict(site(name) => GPUCompiler.JuliaValueRef(value))) # Identical interior relocation identities merge; conflicting metadata does not. dest = interior_module("patch", "first_patch") @@ -967,7 +972,7 @@ end src = interior_module("patch", "second_patch") src_relocs = interior_relocs("patch", Float64) GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs) - @test collect(keys(dest_relocs.interior)) == [("patch", 0)] + @test collect(keys(dest_relocs.sites)) == [site("patch")] dest = interior_module("patch", "first_patch") dest_relocs = interior_relocs("patch", Float64) @@ -987,14 +992,13 @@ end unused = GlobalVariable(src, LLVM.StructType([LLVM.Int64Type(), LLVM.Int64Type()]), "unused_patch") initializer!(unused, ConstantStruct(LLVM.Constant[ConstantInt(0), ConstantInt(1)])) - src_relocs = GPUCompiler.Relocations( - Dict{String,GPUCompiler.RelocationTarget}(), - Dict(("used_patch", 0) => GPUCompiler.JuliaValueRef(Float64), - ("unused_patch", 0) => GPUCompiler.JuliaValueRef(Int64))) + src_relocs = GPUCompiler.Relocations(Dict( + site("used_patch") => GPUCompiler.JuliaValueRef(Float64), + site("unused_patch") => GPUCompiler.JuliaValueRef(Int64))) dest_relocs = GPUCompiler.Relocations() GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; only_needed=true) - @test collect(keys(dest_relocs.interior)) == [("used_patch", 0)] + @test collect(keys(dest_relocs.sites)) == [site("used_patch")] end end From 780c4c9f634a1fbf519019151978a4545d70c68a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Jul 2026 09:44:23 +0200 Subject: [PATCH 29/45] Narrow the relocation lowering API --- src/interface.jl | 6 ++--- src/relocation.jl | 58 ++++++++++++++---------------------------- test/helpers/native.jl | 20 ++++++++++++++- test/native.jl | 6 +++-- 4 files changed, 45 insertions(+), 45 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 117e3b62..b89e4c84 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -512,10 +512,10 @@ end end # HAS_INTEGRATED_CACHE -@public JuliaValueRef, CGlobalRef, RelocationTarget, RelocationSite, Relocations +@public RelocationSite, Relocations @public lower_relocations!, bake_relocations! -@public emit_patchable_relocations!, emit_imported_relocations! -@public resolve_relocation_target, resolved_relocations, supports_relocatable_ir +@public emit_patchable_relocations! +@public resolved_relocations, supports_relocatable_ir @public GPUCompilerCacheToken, cache_owner, cached_results # the method table to use diff --git a/src/relocation.jl b/src/relocation.jl index 9d44334f..2fecf562 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -13,8 +13,6 @@ # Lowering strategy Loader contract Julia analog # `bake_relocations!` none; current-session words baked JIT address folding # `emit_patchable_relocations!` patch named globals after load sysimage gvars -# `emit_imported_relocations!` import declarations before load; ORC absoluteSymbols -# patch definitions afterward # # The producers are `collect_julia_value_relocations!` during IR generation and # `collect_cglobal_relocations!` immediately before backend lowering. Names are globally @@ -94,6 +92,11 @@ patch within its initializer. struct RelocationSite name::String offset::Int + + function RelocationSite(name::String, offset::Int) + offset >= 0 || throw(ArgumentError("relocation offset must be nonnegative")) + new(name, offset) + end end """ @@ -161,7 +164,7 @@ function foreach_relocation(f, mod::LLVM.Module, relocs::Relocations) T isa LLVM.StructType || error("Relocation global '$name' has non-struct initializer $T") size = abi_size(datalayout(mod), T) - 0 <= offset && offset + sizeof(UInt) <= size || + offset + sizeof(UInt) <= size || error("Relocation '$name+$offset' is outside its $size-byte global") end f(site, gv, ref) @@ -402,18 +405,20 @@ end function prune_dead_relocations!(mod::LLVM.Module, relocs::Relocations) mod_gvs = globals(mod) - for name in unique(site.name for site in keys(relocs.sites)) - if !haskey(mod_gvs, name) - for site in collect(keys(relocs.sites)) - site.name == name && delete!(relocs.sites, site) - end - elseif !isdeclaration(mod_gvs[name]) && isempty(uses(mod_gvs[name])) - for site in collect(keys(relocs.sites)) - site.name == name && delete!(relocs.sites, site) - end - erase!(mod_gvs[name]) + dead_names = Set{String}() + for site in keys(relocs.sites) + gv = haskey(mod_gvs, site.name) ? mod_gvs[site.name] : nothing + if gv === nothing || (!isdeclaration(gv) && isempty(uses(gv))) + push!(dead_names, site.name) end end + for site in collect(keys(relocs.sites)) + site.name in dead_names && delete!(relocs.sites, site) + end + for name in dead_names + gv = haskey(mod_gvs, name) ? mod_gvs[name] : nothing + gv === nothing || isdeclaration(gv) || erase!(gv) + end return end @@ -425,7 +430,7 @@ end Backend hook for lowering live relocations before object emission. The default implementation bakes them into the module using the current Julia process. -Backends may instead emit patchable globals or imported slot symbols for their loaders. +Backends may instead emit patchable globals or implement custom lowering for their loaders. Backends using baked lowering must not persist generated code in `cached_results`. Generated code is session-portable only when [`supports_relocatable_ir`](@ref) and the backend preserves @@ -482,31 +487,6 @@ function emit_patchable_relocations!(mod::LLVM.Module, relocs::Relocations) return end -""" - emit_imported_relocations!(mod, relocs) - -Prepare relocations for a loader that imports declaration symbols before loading an object. - -Every declaration remains an external word-sized declaration. The loader must define its -symbol to point at a cell containing [`resolve_relocation_target`](@ref), and keep the cell -and any referenced Julia value alive while the code is executable. Sites in definitions -cannot be imported before load and must always be written afterward. -""" -function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) - used = GlobalVariable[] - foreach_relocation(mod, relocs) do site, gv, _ - if isdeclaration(gv) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, false) - else - push!(used, gv) - end - end - isempty(used) || set_used!(mod, used...) - return -end - ## Introspection function referenced_object(value, relocs::Relocations) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 08f9fa52..781b6a7e 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -24,11 +24,29 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint + +# ORC needs declarations imported before loading and sites in definitions patched afterward. +# Keep that policy local to the JIT test back-end. +function emit_imported_relocations!(mod::LLVM.Module, relocs::GPUCompiler.Relocations) + used = GlobalVariable[] + GPUCompiler.foreach_relocation(mod, relocs) do _, gv, _ + if isdeclaration(gv) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + else + push!(used, gv) + end + end + isempty(used) || set_used!(mod, used...) + return +end + function GPUCompiler.lower_relocations!(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, relocs::GPUCompiler.Relocations) if job.config.params.jit - GPUCompiler.emit_imported_relocations!(mod, relocs) + emit_imported_relocations!(mod, relocs) else invoke(GPUCompiler.lower_relocations!, Tuple{CompilerJob,LLVM.Module,GPUCompiler.Relocations}, job, mod, relocs) diff --git a/test/native.jl b/test/native.jl index 04957dab..70d267e7 100644 --- a/test/native.jl +++ b/test/native.jl @@ -738,6 +738,8 @@ end end @testset "relocation target resolution" begin + @test_throws ArgumentError GPUCompiler.RelocationSite("invalid", -1) + sym = :relocation_target_probe @test GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(sym)) == UInt(pointer_from_objref(sym)) @@ -870,7 +872,7 @@ end site = only(keys(relocs.sites)) @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$(site.name) = external global i64", string(mod)) - GPUCompiler.emit_imported_relocations!(mod, relocs) + Native.emit_imported_relocations!(mod, relocs) @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) @test isdeclaration(globals(mod)[site.name]) @test occursin("@$(site.name) = external global i64", string(mod)) @@ -885,7 +887,7 @@ end relocs = GPUCompiler.Relocations() @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) site = only(keys(relocs.sites)) - GPUCompiler.emit_imported_relocations!(mod, relocs) + Native.emit_imported_relocations!(mod, relocs) @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) @test isdeclaration(globals(mod)[site.name]) @test occursin("@$(site.name) = external global i64", string(mod)) From 59bc80c8a7aa08f2d1d1b2fed80a7e8cacfcc1d9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Jul 2026 10:38:14 +0200 Subject: [PATCH 30/45] Clean-ups. --- src/interface.jl | 6 ------ src/mcgen.jl | 32 ++++++++++++++------------------ src/relocation.jl | 47 ++++++++++++++++++++++++++++------------------- 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index b89e4c84..4f4509af 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -429,12 +429,6 @@ the check-and-compile sequence (all back-ends already do, e.g. `mtlfunction_lock """ function cached_results end -# Relocation types and the backend lowering hook live in relocation.jl. -# -# Job results attached to CodeInstances are serialized into package images. Store only -# session-portable data there. Generated code is portable iff `supports_relocatable_ir()` -# and the backend preserves relocations with patchable- or imported-symbol lowering. Backends -# using baked lowering must keep generated code in a session-local cache or recompile it. @static if HAS_INTEGRATED_CACHE """ diff --git a/src/mcgen.jl b/src/mcgen.jl index fdb96349..0dda9ecc 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -1,31 +1,27 @@ # machine code generation -# GlobalOpt/DCE cleanup, run before slot collection and again after lowering. -function run_cleanup_pipeline!(@nospecialize(job::CompilerJob), mod::LLVM.Module) - @dispose pb=NewPMPassBuilder() begin - add!(pb, RecomputeGlobalsAAPass()) - add!(pb, GlobalOptPass()) - add!(pb, GlobalDCEPass()) - add!(pb, StripDeadPrototypesPass()) - run!(pb, mod, llvm_machine(job.config.target)) - end - return -end - -# Final preparations for the module to be compiled to machine code. These passes should not -# be run when e.g. compiling to write to disk. +# Finalize the module for backend emission by collecting and lowering all live relocations. function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, relocs::Relocations=Relocations()) # Clean up first so only live relocations get lowered. - run_cleanup_pipeline!(job, mod) + function cleanup() + @dispose pb=NewPMPassBuilder() begin + add!(pb, RecomputeGlobalsAAPass()) + add!(pb, GlobalOptPass()) + add!(pb, GlobalDCEPass()) + add!(pb, StripDeadPrototypesPass()) + run!(pb, mod, llvm_machine(job.config.target)) + end + end + cleanup() prune_dead_relocations!(mod, relocs) collect_cglobal_relocations!(mod, relocs) lower_relocations!(job, mod, relocs) - # Fold constants exposed by eager lowering, and discard slots made dead by either - # lowering strategy. - run_cleanup_pipeline!(job, mod) + # Fold constants exposed by eager lowering, and discard slots made dead by + # either lowering strategy. + cleanup() prune_dead_relocations!(mod, relocs) has_unresolved_cglobal_loads(mod, relocs) && diff --git a/src/relocation.jl b/src/relocation.jl index 2fecf562..a0c41db2 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -1,27 +1,31 @@ # Relocation model # # A relocation is a site => target pair. Targets are Julia object identities -# (`JuliaValueRef`, like codegen's identity-keyed `global_targets`) or named libjulia -# globals (`CGlobalRef`, like JuliaVariables). Every site names a word at a byte offset in a -# global. Dedicated sites use offset zero in a word-sized declaration (like -# `jl_sysimg_gvars`); other sites are inside defined globals (like -# `gctags_list`/`relocs_list`). Declarations can be baked, patched, or imported; sites in -# definitions can only be baked or patched. +# (`JuliaValueRef`, like codegen's identity-keyed `global_targets`) or named +# libjulia globals (`CGlobalRef`, like JuliaVariables). Every site names a word +# at a byte offset in a global. Dedicated sites use offset zero in a word-sized +# declaration (like `jl_sysimg_gvars`); other sites are inside defined globals +# (like `gctags_list`/`relocs_list`). Declarations can be baked, patched, or +# imported; sites in definitions can only be baked or patched. # # produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) # -# Lowering strategy Loader contract Julia analog -# `bake_relocations!` none; current-session words baked JIT address folding -# `emit_patchable_relocations!` patch named globals after load sysimage gvars +# Lowering strategy Loader contract Julia analog +# ----------------- --------------- ------------ +# `bake_relocations!` none; current-session words baked JIT address folding +# `emit_patchable_relocations!` patch named globals after load sysimage gvars # # The producers are `collect_julia_value_relocations!` during IR generation and -# `collect_cglobal_relocations!` immediately before backend lowering. Names are globally -# unique and assigned once, so linking can merge IR and metadata without renaming. +# `collect_cglobal_relocations!` immediately before backend lowering. Names are +# globally unique and assigned once, so linking can merge IR and metadata +# without renaming. # -# Generated code is portable only when `supports_relocatable_ir()` and the backend preserves -# these relocations for its loader. The default lowering bakes current-session values. +# Generated code is portable only when `supports_relocatable_ir()` and the +# backend preserves these relocations for its loader. The default lowering bakes +# current-session values. -## Targets + +## targets """ JuliaValueRef(value) @@ -79,7 +83,8 @@ function resolve_relocation_target(target::CGlobalRef) return unsafe_load(address) end -## The table + +## the table """ RelocationSite(name, offset) @@ -172,7 +177,8 @@ function foreach_relocation(f, mod::LLVM.Module, relocs::Relocations) return end -## Producers + +## producers function collect_julia_value_relocations!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) @@ -364,7 +370,8 @@ function has_unresolved_cglobal_loads(mod::LLVM.Module, relocs::Relocations) return false end -## Bookkeeping + +## bookkeeping function link_relocatable!(dest_mod::LLVM.Module, dest_relocs::Relocations, src_mod::LLVM.Module, src_relocs::Relocations; @@ -422,7 +429,8 @@ function prune_dead_relocations!(mod::LLVM.Module, relocs::Relocations) return end -## Lowering + +## lowering """ lower_relocations!(job, mod, relocs) @@ -487,7 +495,8 @@ function emit_patchable_relocations!(mod::LLVM.Module, relocs::Relocations) return end -## Introspection + +## introspection function referenced_object(value, relocs::Relocations) # This is best-effort: optimized shapes fall back to the unknown-binding error path. From 6e57ef6b472da728c3b04b0406382830416266f7 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Fri, 17 Jul 2026 10:44:46 +0200 Subject: [PATCH 31/45] Account for Julia 1.10 boxed constants in PTX tests --- test/ptx.jl | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/ptx.jl b/test/ptx.jl index 4f44c0af..b04ccadf 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -99,7 +99,14 @@ end end end ir = sprint(io->PTX.code_llvm(io, mod.consume, Tuple{Bool,Int32}; dump_module=true)) - @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) + # As with Symbol literals above, Julia 1.10 may bake the type pointer before + # GPUCompiler can represent it as a relocation. + @static if VERSION >= v"1.11" + @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) + else + @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) || + occursin("inttoptr", ir) + end end @testset "kernel functions" begin From 901a5a577e86cd6141355a7fffad30175d270616 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 12:07:28 +0200 Subject: [PATCH 32/45] Select relocation lowering by back-end trait Add a `relocation_lowering(job)` trait choosing `:bake` (default), `:patch`, or `:import`. `:bake` resolves host references into the IR eagerly during `emit_llvm` (where `relocate_gvs!` used to run), so the `:llvm` result is execution-ready and back-ends that drive `emit_asm(job, ir, format)` directly (Metal.jl) keep working; a restored three-argument `emit_asm` forwards to the metadata-carrying method. Non-baking strategies keep references symbolic for their loader and lower them at object emission. `lower_relocations!` becomes an internal dispatcher on the trait, subsuming the per-back-end override; the previously test-local import lowering moves in as `emit_imported_relocations!`. Also add `Base.copy(::Relocations)` for loaders that bake from cached metadata. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/driver.jl | 34 +++++++++++++-- src/interface.jl | 27 ++++++++++-- src/relocation.jl | 93 ++++++++++++++++++++++++++++++------------ test/helpers/native.jl | 31 ++------------ test/native.jl | 4 +- 5 files changed, 127 insertions(+), 62 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index 167f5944..91644a1d 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -51,9 +51,11 @@ export compile Compile a `job` to one of the following formats as specified by the `target` argument: `:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. -The returned `relocations` metadata is final only for `:asm` and `:obj`, after runtime -globals have been collected and the backend has lowered every live slot. The `:llvm` result -still contains symbolic Julia-value slots and may contain raw `jl_*` runtime references. +For the default `:bake` [`relocation_lowering`](@ref) strategy the `:llvm` result is +execution-ready: Julia-value references are resolved into the IR during `emit_llvm` (it may +still contain raw `jl_*` runtime references, collected and resolved at object emission). A +back-end that defers relocation lowering keeps those references symbolic in the `:llvm` +result and returns final `relocations` metadata only for `:asm`/`:obj`. """ function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing @@ -337,6 +339,16 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) + # Baking back-ends (the default) resolve host references into the IR here — + # before optimization, where `relocate_gvs!` used to run — so the optimizer sees + # concrete values and the `:llvm` result is execution-ready. Other strategies + # keep relocations symbolic for their loader, lowering them at object emission. + bake_relocations = relocation_lowering(job) === :bake + if bake_relocations + prune_dead_relocations!(ir, relocations) + bake_relocations!(ir, relocations) + end + if job.config.optimize @tracepoint "optimization" begin optimize!(job, ir, relocations; job.config.opt_level) @@ -361,6 +373,15 @@ const __llvm_initialized = Ref(false) end end + # Runtime linking during optimization can introduce further relocations (GC + # lowering materializes runtime calls that reference Julia globals). Bake those + # too, so a baking back-end's module is fully resolved before object emission + # even when the caller drives `emit_asm` without relocation metadata (Metal.jl). + if bake_relocations && !isempty(relocations.sites) + prune_dead_relocations!(ir, relocations) + bake_relocations!(ir, relocations) + end + if job.config.cleanup @tracepoint "clean-up" begin @dispose pb=NewPMPassBuilder() begin @@ -415,6 +436,13 @@ const __llvm_initialized = Ref(false) return ir, (; entry, compiled, relocations) end +# Forwarding method for back-ends that drive emission directly (e.g. Metal.jl's +# `emit_llvm` + `emit_asm`). Correct for baking back-ends: their relocations were resolved +# during `emit_llvm`, so no metadata needs threading through here. +emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, + format::LLVM.API.LLVMCodeGenFileType) = + emit_asm(job, ir, Relocations(), format) + @locked function emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, relocs::Relocations, format::LLVM.API.LLVMCodeGenFileType) # NOTE: strip after validation to get better errors diff --git a/src/interface.jl b/src/interface.jl index 4f4509af..0feafd17 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -302,6 +302,27 @@ can_vectorize(@nospecialize(job::CompilerJob)) = false # Should emit PTLS lookup that can be relocated dump_native(@nospecialize(job::CompilerJob)) = false +# How this back-end lowers relocations (host references) before object emission. One of: +# +# - `:bake` (default): resolve the references in the current session and materialize them +# into the IR. The `:llvm` result is execution-ready and no loader is needed. Baked code +# embeds session-local addresses, so it must not be persisted in `cached_results`. +# +# - `:patch`: keep references symbolic through optimization and emit every site as a +# patchable (null-init, externally-initialized) definition. The loader writes each +# resolved word at `global + offset` after loading the object. +# +# - `:import`: keep references symbolic and leave declaration slots external for the loader +# to resolve at link time (e.g. ORC `absoluteSymbols`); interior definition sites are +# still patched after loading. +# +# The value also fixes lowering *timing*: `:bake` resolves eagerly during `emit_llvm`, +# other strategies defer to object emission. `:patch`/`:import` loaders must keep the +# `roots` returned by [`resolved_relocations`](@ref) alive while the code can run. Generated +# code is session-portable only when [`supports_relocatable_ir`](@ref) and a non-baking +# strategy is used. +relocation_lowering(@nospecialize(job::CompilerJob)) = :bake + # the Julia module to look up target-specific runtime functions in (this includes both # target-specific functions from the GPU runtime library, like `malloc`, but also # replacements functions for operations like `Base.sin`) @@ -506,9 +527,9 @@ end end # HAS_INTEGRATED_CACHE -@public RelocationSite, Relocations -@public lower_relocations!, bake_relocations! -@public emit_patchable_relocations! +@public RelocationSite, Relocations, relocation_lowering +@public bake_relocations!, emit_patchable_relocations!, emit_imported_relocations! +@public prune_dead_relocations! @public resolved_relocations, supports_relocatable_ir @public GPUCompilerCacheToken, cache_owner, cached_results diff --git a/src/relocation.jl b/src/relocation.jl index a0c41db2..3807f48f 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -5,24 +5,27 @@ # libjulia globals (`CGlobalRef`, like JuliaVariables). Every site names a word # at a byte offset in a global. Dedicated sites use offset zero in a word-sized # declaration (like `jl_sysimg_gvars`); other sites are inside defined globals -# (like `gctags_list`/`relocs_list`). Declarations can be baked, patched, or -# imported; sites in definitions can only be baked or patched. +# (like `gctags_list`/`relocs_list`). A declaration can be baked, patched, or +# resolved by a loader; a word inside a definition can only be baked or patched. # -# produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower (at emit_asm) +# produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower # -# Lowering strategy Loader contract Julia analog -# ----------------- --------------- ------------ -# `bake_relocations!` none; current-session words baked JIT address folding -# `emit_patchable_relocations!` patch named globals after load sysimage gvars +# The back-end's `relocation_lowering` strategy picks how (and when) sites lower: # -# The producers are `collect_julia_value_relocations!` during IR generation and -# `collect_cglobal_relocations!` immediately before backend lowering. Names are -# globally unique and assigned once, so linking can merge IR and metadata -# without renaming. +# Strategy Lowering Loader contract Julia analog +# -------- -------- --------------- ------------ +# `:bake` `bake_relocations!` none; current-session words baked JIT address folding +# `:patch` `emit_patchable_relocations!` patch every site after load sysimage gvars +# `:import` `emit_imported_relocations!` resolve declarations at link time sysimg symbol import # -# Generated code is portable only when `supports_relocatable_ir()` and the -# backend preserves these relocations for its loader. The default lowering bakes -# current-session values. +# Back-ends may also implement custom lowerings on top of these primitives (see the native +# test helper's JIT loader). The producers are `collect_julia_value_relocations!` during IR +# generation and `collect_cglobal_relocations!` immediately before back-end lowering. Names +# are globally unique and assigned once, so linking can merge IR and metadata without +# renaming. +# +# Generated code is portable only when `supports_relocatable_ir()` and a non-baking strategy +# preserves these relocations for its loader. The default (`:bake`) resolves in-session. ## targets @@ -116,6 +119,11 @@ end Relocations() = Relocations(Dict{RelocationSite,RelocationTarget}()) +# A loader that bakes from cached metadata consumes the sites (baking empties them), so it +# needs a fresh copy per link. The site keys and targets are immutable, so a shallow copy of +# the dict suffices. +Base.copy(relocs::Relocations) = Relocations(copy(relocs.sites)) + """ resolved_relocations(relocs) @@ -432,23 +440,32 @@ end ## lowering -""" - lower_relocations!(job, mod, relocs) - -Backend hook for lowering live relocations before object emission. - -The default implementation bakes them into the module using the current Julia process. -Backends may instead emit patchable globals or implement custom lowering for their loaders. - -Backends using baked lowering must not persist generated code in `cached_results`. Generated -code is session-portable only when [`supports_relocatable_ir`](@ref) and the backend preserves -all relocations for its loader. -""" +# Lower live relocations before object emission, dispatching on the back-end's +# `relocation_lowering` strategy. Internal: back-ends select a strategy through the trait +# rather than overriding this. function lower_relocations!(@nospecialize(job::CompilerJob), mod::LLVM.Module, relocs::Relocations) - bake_relocations!(mod, relocs) + strategy = relocation_lowering(job) + if strategy === :bake + bake_relocations!(mod, relocs) + elseif strategy === :patch + emit_patchable_relocations!(mod, relocs) + elseif strategy === :import + emit_imported_relocations!(mod, relocs) + else + error("Unknown relocation lowering strategy :$strategy") + end + return end +""" + bake_relocations!(mod, relocs) + +Resolve every site in the current Julia process and materialize the resulting words into +the IR, leaving `relocs` empty. The module is execution-ready but embeds session-local +addresses, so it must not be persisted across sessions. Dead sites must be dropped first +with [`prune_dead_relocations!`](@ref); baking errors on a site whose global is gone. +""" function bake_relocations!(mod::LLVM.Module, relocs::Relocations) foreach_relocation(mod, relocs) do site, gv, ref if isdeclaration(gv) @@ -495,6 +512,28 @@ function emit_patchable_relocations!(mod::LLVM.Module, relocs::Relocations) return end +""" + emit_imported_relocations!(mod, relocs) + +Leave declaration slots external so a JIT loader resolves them at link time (e.g. ORC +`absoluteSymbols`); a word inside a definition cannot be imported, so those sites stay +`llvm.used` for the loader to patch after loading. Requires a per-object symbol namespace. +""" +function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) + used = GlobalVariable[] + foreach_relocation(mod, relocs) do _, gv, _ + if isdeclaration(gv) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + else + push!(used, gv) + end + end + isempty(used) || set_used!(mod, used...) + return +end + ## introspection diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 781b6a7e..8ae67bb3 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -25,33 +25,10 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint -# ORC needs declarations imported before loading and sites in definitions patched afterward. -# Keep that policy local to the JIT test back-end. -function emit_imported_relocations!(mod::LLVM.Module, relocs::GPUCompiler.Relocations) - used = GlobalVariable[] - GPUCompiler.foreach_relocation(mod, relocs) do _, gv, _ - if isdeclaration(gv) - constant!(gv, false) - linkage!(gv, LLVM.API.LLVMExternalLinkage) - extinit!(gv, false) - else - push!(used, gv) - end - end - isempty(used) || set_used!(mod, used...) - return -end - -function GPUCompiler.lower_relocations!(@nospecialize(job::NativeCompilerJob), - mod::LLVM.Module, - relocs::GPUCompiler.Relocations) - if job.config.params.jit - emit_imported_relocations!(mod, relocs) - else - invoke(GPUCompiler.lower_relocations!, - Tuple{CompilerJob,LLVM.Module,GPUCompiler.Relocations}, job, mod, relocs) - end -end +# The JIT mode drives an ORC loader (see `load`), so keep relocations symbolic and import +# them at link time; otherwise bake in-session like a plain compilation. +GPUCompiler.relocation_lowering(@nospecialize(job::NativeCompilerJob)) = + job.config.params.jit ? :import : :bake function GPUCompiler.mcgen(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, format=LLVM.API.LLVMAssemblyFile) diff --git a/test/native.jl b/test/native.jl index 70d267e7..7c39c973 100644 --- a/test/native.jl +++ b/test/native.jl @@ -872,7 +872,7 @@ end site = only(keys(relocs.sites)) @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) @test occursin("@$(site.name) = external global i64", string(mod)) - Native.emit_imported_relocations!(mod, relocs) + GPUCompiler.emit_imported_relocations!(mod, relocs) @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) @test isdeclaration(globals(mod)[site.name]) @test occursin("@$(site.name) = external global i64", string(mod)) @@ -887,7 +887,7 @@ end relocs = GPUCompiler.Relocations() @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) site = only(keys(relocs.sites)) - Native.emit_imported_relocations!(mod, relocs) + GPUCompiler.emit_imported_relocations!(mod, relocs) @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) @test isdeclaration(globals(mod)[site.name]) @test occursin("@$(site.name) = external global i64", string(mod)) From 30fb7729c8e96451dd9f9f78e4256a19fd4064f9 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 12:08:53 +0200 Subject: [PATCH 33/45] Let CGlobalRef name a library and symbol Add a `library::Union{Nothing,String}` field to `CGlobalRef`. The default `nothing` keeps the current behavior (libjulia global via `jl_cglobal`'s process-wide lookup); an explicit library is loaded and searched with Libdl. Both fields are plain data, so serialized relocation metadata stays portable. Drop the `String(symbol)` conversion in resolution: `jl_cglobal` accepts the symbol directly. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/relocation.jl | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/relocation.jl b/src/relocation.jl index 3807f48f..4086cf26 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -49,14 +49,19 @@ struct JuliaValueRef end """ - CGlobalRef(symbol) + CGlobalRef(symbol, library=nothing) -A named libjulia C data global. Resolution returns the word stored in that global in the -current Julia process. +A named C data global. With `library === nothing` (the default) it names a libjulia global, +resolved with `jl_cglobal`'s process-wide symbol lookup; an explicit `library` names a +shared object to load and look the symbol up in. Resolution returns the word stored in that +global in the current Julia process. Both fields are plain data, so serialized metadata +stays portable. """ struct CGlobalRef symbol::Symbol + library::Union{Nothing,String} end +CGlobalRef(symbol::Symbol) = CGlobalRef(symbol, nothing) """ RelocationTarget @@ -67,7 +72,8 @@ A serializable target for a relocated word: either a [`JuliaValueRef`](@ref) or const RelocationTarget = Union{JuliaValueRef,CGlobalRef} same_relocation_target(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value -same_relocation_target(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol +same_relocation_target(a::CGlobalRef, b::CGlobalRef) = + a.symbol === b.symbol && a.library == b.library same_relocation_target(::RelocationTarget, ::RelocationTarget) = false """ @@ -82,8 +88,14 @@ function resolve_relocation_target(target::JuliaValueRef) end end function resolve_relocation_target(target::CGlobalRef) - address = ccall(:jl_cglobal, Any, (Any, Any), String(target.symbol), UInt) - return unsafe_load(address) + if target.library === nothing + # `jl_cglobal` accepts the symbol directly and does the process-wide `jl_dlfind`. + address = ccall(:jl_cglobal, Any, (Any, Any), target.symbol, UInt) + return unsafe_load(address) + end + handle = Libdl.dlopen(target.library) + address = Libdl.dlsym(handle, target.symbol) + return unsafe_load(Ptr{UInt}(address)) end From 49cbd51bb8b76bcfa92b2ea113a0482d42d2c981 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 12:09:11 +0200 Subject: [PATCH 34/45] Explain the GPULinkRuntime by-name pass reference Only GPULinkRuntime is added by name (to reuse the instance holding the Relocations owned by optimize!); the sibling passes are stateless and constructed fresh. Spell that out at the call site. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/optim.jl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/optim.jl b/src/optim.jl index 02bf7c77..8522b707 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -326,8 +326,11 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op add!(fpm, GPULowerGCFramePass(job)) end if job.config.libraries - # The pass builder resolves this name to the registered instance, which - # captures the Relocations object owned by optimize!. + # Add GPULinkRuntime *by name* so the pass builder resolves it to the instance + # registered in `optimize!`: that instance holds the `Relocations` object owned + # by `optimize!`, and relinking the runtime here must merge new sites into it. + # The other two passes are stateless with respect to that object, so fresh + # job-capturing instances are equivalent. add!(mpm, "GPULinkRuntime") add!(mpm, GPULinkLibrariesPass(job)) add!(mpm, GPUFinishRuntimeIntrinsicsPass(job)) From 4351a0a241fcb7084709a1a5920260f620f4b932 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 12:13:52 +0200 Subject: [PATCH 35/45] Gate relocation tests on supports_relocatable_ir() Whether relocations are exposed symbolically is the runtime capability `supports_relocatable_ir()` (jl_get_llvm_gvs_globals was backported to 1.11/1.12 point releases), not a fixed version. Replace the `VERSION >=` gates in the relocation tests with the probe. The tests that observe symbolic slots surviving to `:llvm` now compile with a non-baking back-end (native `jit=true`, a new PTX `patch=true`), since a baking job resolves and folds those slots away before the output. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/helpers/ptx.jl | 14 +++++++++++--- test/native.jl | 11 ++++++----- test/ptx.jl | 19 +++++++++++-------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/test/helpers/ptx.jl b/test/helpers/ptx.jl index 4b00f707..1bf2cbeb 100644 --- a/test/helpers/ptx.jl +++ b/test/helpers/ptx.jl @@ -3,10 +3,18 @@ module PTX using ..GPUCompiler import ..TestRuntime -struct CompilerParams <: AbstractCompilerParams end +struct CompilerParams <: AbstractCompilerParams + patch::Bool + CompilerParams(patch::Bool=false) = new(patch) +end PTXCompilerJob = CompilerJob{PTXCompilerTarget,CompilerParams} +# `patch=true` keeps relocations symbolic (as CUDA.jl does) so tests can observe them +# surviving to the `:llvm`/`:asm` output; otherwise bake in-session like a plain compilation. +GPUCompiler.relocation_lowering(@nospecialize(job::PTXCompilerJob)) = + job.config.params.patch ? :patch : :bake + struct PTXKernelState data::Int64 end @@ -39,14 +47,14 @@ function create_job(@nospecialize(func), @nospecialize(types); cap=v"7.0", ptx=v"6.0", feature_set=:baseline, minthreads=nothing, maxthreads=nothing, blocks_per_sm=nothing, maxregs=nothing, - fastmath=false, + fastmath=false, patch::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = PTXCompilerTarget(; cap, ptx, feature_set, minthreads, maxthreads, blocks_per_sm, maxregs, fastmath) - params = CompilerParams() + params = CompilerParams(patch) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end diff --git a/test/native.jl b/test/native.jl index 7c39c973..9ab80743 100644 --- a/test/native.jl +++ b/test/native.jl @@ -59,7 +59,8 @@ end end end - job, _ = Native.create_job(mod.outer, (Int, Symbol); validate=false) + # a non-baking (JIT) back-end keeps the Symbol reference symbolic in `:llvm`. + job, _ = Native.create_job(mod.outer, (Int, Symbol); validate=false, jit=true) JuliaContext() do ctx ir, meta = GPUCompiler.compile(:llvm, job) @@ -72,7 +73,7 @@ end @test length(other_mis) == 1 @test only(other_mis).def in methods(mod.inner) - if VERSION >= v"1.12" + if GPUCompiler.supports_relocatable_ir() @test length(meta.relocations.sites) == 1 @test only(values(meta.relocations.sites)) isa GPUCompiler.JuliaValueRef end @@ -272,8 +273,8 @@ end site = GPUCompiler.RelocationSite(LLVM.name(gv), 0) @test haskey(lib.relocations.sites, site) end - @static if VERSION >= v"1.12-" - # on older versions, Julia bakes addresses without tagging globals + if GPUCompiler.supports_relocatable_ir() + # otherwise Julia bakes addresses without tagging globals @test used > 0 end end @@ -752,7 +753,7 @@ end end @testset "postponed relocation" begin - if VERSION >= v"1.12" + if GPUCompiler.supports_relocatable_ir() mod = @eval module $(gensym()) f() = UInt(pointer_from_objref(:relocation_probe)) end diff --git a/test/ptx.jl b/test/ptx.jl index b04ccadf..612d912f 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -37,12 +37,13 @@ end mod = @eval module $(gensym()) kernel(name::Symbol) = name === :var ? 1 : 2 end + # `patch=true` keeps the slot symbolic (a baking back-end would resolve and fold it away). ir = sprint() do io - PTX.code_llvm(io, mod.kernel, Tuple{Symbol}; dump_module=true) + PTX.code_llvm(io, mod.kernel, Tuple{Symbol}; dump_module=true, patch=true) end - # Julia 1.10 may bake the Symbol pointer into the generated IR before exposing it to - # GPUCompiler. Newer versions provide a symbolic global that we can preserve and relocate. - @static if VERSION >= v"1.11" + # Julia bakes the Symbol pointer into the IR before exposing it to GPUCompiler when it + # can't emit relocatable global metadata; otherwise it stays a symbolic slot we preserve. + if GPUCompiler.supports_relocatable_ir() @test occursin("@jl_sym_var_", ir) else @test occursin("@jl_sym_var_", ir) || occursin("inttoptr", ir) @@ -98,10 +99,12 @@ end return 0.0 end end - ir = sprint(io->PTX.code_llvm(io, mod.consume, Tuple{Bool,Int32}; dump_module=true)) - # As with Symbol literals above, Julia 1.10 may bake the type pointer before - # GPUCompiler can represent it as a relocation. - @static if VERSION >= v"1.11" + # `patch=true` keeps the interior header relocation symbolic (externally_initialized). + ir = sprint(io->PTX.code_llvm(io, mod.consume, Tuple{Bool,Int32}; + dump_module=true, patch=true)) + # As with Symbol literals above, Julia bakes the type pointer when it can't emit + # relocatable global metadata; otherwise GPUCompiler represents it as a relocation. + if GPUCompiler.supports_relocatable_ir() @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) else @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) || From 4c3e8b0f3035d95fc2f17ce6964e58b05a873290 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 12:19:32 +0200 Subject: [PATCH 36/45] Test the relocation lowering strategies end to end Add coverage for the paths the strategy trait introduces: - eager `:bake` default: `compile(:llvm)` leaves no relocation metadata; - `:patch`: load an object, patch every site at `global + offset`, execute, and check the null-init / extinit / llvm.used shape (native, plus a PTX `.global` assertion); - validation error paths in `foreach_relocation` and `link_relocatable!`, and a direct `prune_dead_relocations!` test; - a deferred (Mock Enzyme) child whose relocations must merge into the parent. The native test back-end gains a `:patch` mode alongside its `:import` JIT mode. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/helpers/native.jl | 18 ++--- test/native.jl | 150 +++++++++++++++++++++++++++++++++++++++++ test/ptx.jl | 18 +++++ 3 files changed, 178 insertions(+), 8 deletions(-) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 8ae67bb3..0a54c50c 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -11,10 +11,11 @@ struct CompilerParams <: AbstractCompilerParams entry_safepoint::Bool method_table jit::Bool + patch::Bool CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table, - jit::Bool=false) = - new(entry_safepoint, method_table, jit) + jit::Bool=false, patch::Bool=false) = + new(entry_safepoint, method_table, jit, patch) end module Runtime end @@ -25,14 +26,15 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint -# The JIT mode drives an ORC loader (see `load`), so keep relocations symbolic and import -# them at link time; otherwise bake in-session like a plain compilation. +# Both non-baking modes drive an ORC loader (see `load`): `jit` imports declarations at +# link time, `patch` emits patchable definitions to write after loading. Plain jobs bake. GPUCompiler.relocation_lowering(@nospecialize(job::NativeCompilerJob)) = - job.config.params.jit ? :import : :bake + job.config.params.patch ? :patch : + job.config.params.jit ? :import : :bake function GPUCompiler.mcgen(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, format=LLVM.API.LLVMAssemblyFile) - if job.config.params.jit + if job.config.params.jit || job.config.params.patch target = job.config.target @dispose tm=JITTargetMachine(GPUCompiler.llvm_triple(target), target.cpu, target.features) begin @@ -46,11 +48,11 @@ end function create_job(@nospecialize(func), @nospecialize(types); entry_safepoint::Bool=false, method_table=test_method_table, - jit::Bool=false, kwargs...) + jit::Bool=false, patch::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = NativeCompilerTarget(;jlruntime=true) - params = CompilerParams(entry_safepoint, method_table, jit) + params = CompilerParams(entry_safepoint, method_table, jit, patch) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end diff --git a/test/native.jl b/test/native.jl index 9ab80743..4e7caf20 100644 --- a/test/native.jl +++ b/test/native.jl @@ -812,6 +812,115 @@ end end end +@testset "eager relocation baking" begin + # The default (baking) back-end resolves host references into the IR during `emit_llvm`, + # so the `:llvm` result carries no relocation metadata and needs no loader — the contract + # that lets Metal.jl and unmodified AllocCheck consume it directly. + mod = @eval module $(gensym()) + probe() = UInt(pointer_from_objref(:eager_probe)) + end + job, _ = Native.create_job(mod.probe, Tuple{}) + JuliaContext() do ctx + ir, meta = GPUCompiler.compile(:llvm, job) + @test isempty(meta.relocations.sites) + # nothing is left for a loader to patch or import + @test !any(GPUCompiler.isextinit, globals(ir)) + end +end + +@testset "patchable relocation" begin + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + f() = UInt(pointer_from_objref(:patch_probe)) + end + job, _ = Native.create_job(mod.f, Tuple{}; patch=true) + JuliaContext() do ctx + obj, meta = GPUCompiler.compile(:obj, job) + relocs = meta.relocations + @test !isempty(relocs.sites) + + # every declaration slot became a null-init, externally-initialized definition + # kept alive by `llvm.used`; the loader patches each site after loading. + @test haskey(globals(meta.ir), "llvm.used") + for site in keys(relocs.sites) + gv = globals(meta.ir)[site.name] + @test !isdeclaration(gv) + @test isextinit(gv) + @test !isconstant(gv) + @test linkage(gv) == LLVM.API.LLVMExternalLinkage + @test LLVM.isnull(initializer(gv)) + end + + bytes = Vector{UInt8}(codeunits(obj)) + entry = LLVM.name(meta.entry) + expected = GPUCompiler.resolve_relocation_target(only(values(relocs.sites))) + fptr, keepalive = Native.load(bytes, entry, relocs, meta.ir) + try + GC.@preserve keepalive begin + @test ccall(fptr, UInt, ()) == expected + end + finally + dispose(first(keepalive)) + end + end + end +end + +@testset "relocation validation errors" begin + JuliaContext() do ctx + word() = GPUCompiler.relocation_word_type() + nop = (_site, _gv, _ref) -> nothing + ref = GPUCompiler.JuliaValueRef(:probe) + reloc(name, offset=0) = + GPUCompiler.Relocations(Dict(GPUCompiler.RelocationSite(name, offset) => ref)) + + # a site whose global is absent from the module + mod = LLVM.Module("errors") + @test_throws "Missing relocation global" GPUCompiler.foreach_relocation( + nop, mod, reloc("absent")) + + # a declaration slot may only carry a zero offset + mod = LLVM.Module("errors") + GlobalVariable(mod, word(), "slot") + @test_throws "nonzero offset" GPUCompiler.foreach_relocation( + nop, mod, reloc("slot", 8)) + + # a declaration slot must be word-sized + mod = LLVM.Module("errors") + GlobalVariable(mod, LLVM.Int32Type(), "narrow") + @test_throws "has size" GPUCompiler.foreach_relocation(nop, mod, reloc("narrow")) + + # an interior site must land within its global + mod = LLVM.Module("errors") + gv = GlobalVariable(mod, LLVM.StructType([LLVM.Int64Type(), LLVM.Int64Type()]), "box") + initializer!(gv, ConstantStruct(LLVM.Constant[ConstantInt(0), ConstantInt(0)])) + @test_throws "outside its" GPUCompiler.foreach_relocation(nop, mod, reloc("box", 16)) + end +end + +@testset "prune dead relocations" begin + JuliaContext() do ctx + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + mod = parse(LLVM.Module, """ + @live = external global i64 + @dead = internal global { i64, i64 } { i64 0, i64 0 } + define i64 @use() { + %v = load i64, $(ptr("i64")) @live + ret i64 %v + }""") + site(name, offset=0) = GPUCompiler.RelocationSite(name, offset) + relocs = GPUCompiler.Relocations(Dict( + site("live") => GPUCompiler.JuliaValueRef(:live), + site("dead") => GPUCompiler.JuliaValueRef(:dead), # unused definition + site("absent") => GPUCompiler.JuliaValueRef(:absent), # global already gone + )) + GPUCompiler.prune_dead_relocations!(mod, relocs) + @test collect(keys(relocs.sites)) == [site("live")] + @test haskey(globals(mod), "live") # a used declaration survives + @test !haskey(globals(mod), "dead") # the dead definition is erased + end +end + @testset "cglobal relocation" begin # JIT-private symbols like `jl_get_pgcstack_resolved` (JuliaLang/julia#61527) cannot # be looked up using `jl_cglobal`, so we should only resolve bindings that are @@ -1002,6 +1111,22 @@ end GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; only_needed=true) @test collect(keys(dest_relocs.sites)) == [site("used_patch")] + + # A shared site whose source global is missing is an inconsistency, as is a source + # definition with no matching destination global to link against. + dest = slot_module("slot", "first") + dest_relocs = slot_relocs("slot", :shared) + src = LLVM.Module("empty") + src_relocs = slot_relocs("slot", :shared) + @test_throws "Missing source relocation global" GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) + + dest = LLVM.Module("empty") + dest_relocs = interior_relocs("patch", Float64) + src = interior_module("patch", "second_patch") + src_relocs = interior_relocs("patch", Float64) + @test_throws "Missing destination relocation global" GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) end end @@ -1356,6 +1481,31 @@ end @test occursin("call void @julia_kernel", ir) end +@testset "Mock Enzyme deferred relocations" begin + # A deferred child that references a Julia value produces its own relocations; those + # must merge into the parent's metadata when the child module is linked in. + mod = @eval module $(gensym()) + import ..Enzyme + child(sym::Symbol) = sym === :deferred_reloc ? 1 : 2 + function parent(sym::Symbol) + ptr = Enzyme.deferred_codegen(typeof(child), Tuple{Symbol}) + return ccall(ptr, Int, (Symbol,), sym) + end + end + + # a non-baking job keeps the merged relocation symbolic so we can inspect it. + job, _ = Native.create_job(mod.parent, (Symbol,); jit=true, validate=false) + JuliaContext() do ctx + ir, meta = GPUCompiler.compile(:llvm, job) + @test !occursin("deferred_codegen", string(ir)) + if GPUCompiler.supports_relocatable_ir() + @test any(values(meta.relocations.sites)) do ref + ref isa GPUCompiler.JuliaValueRef && ref.value === :deferred_reloc + end + end + end +end + @testset "stack allocation intrinsic" begin mod = @eval module $(gensym()) import ..GPUCompiler diff --git a/test/ptx.jl b/test/ptx.jl index 612d912f..de773404 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -240,6 +240,24 @@ if :NVPTX in LLVM.backends() end end +@testset "patchable relocation" begin + # A patching back-end (like CUDA.jl) keeps relocations symbolic; the patchable box must + # survive lowering into the generated PTX as a `.global` for the loader to write. + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + @noinline produce(cond::Bool, value::Int32) = cond ? value : 1.5 + function consume(cond::Bool, value::Int32) + x = produce(cond, value) + x isa Float64 && return x + return 0.0 + end + end + ptx = sprint(io->PTX.code_native(io, mod.consume, Tuple{Bool,Int32}; + dump_module=true, patch=true)) + @test occursin(r"\.global .*_box", ptx) + end +end + @testset "child functions" begin # we often test using @noinline child functions, so test whether these survive # (despite not having side-effects) From f1db3cdc22f9b14413aa42be69a33b818a94509f Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 12:19:58 +0200 Subject: [PATCH 37/45] Bump to 2.2.0 for the Relocations API The relocation metadata now surfaced through `compile`/`emit_asm` and the `relocation_lowering` trait is additive: back-ends that don't opt in keep the prior baking behavior. Minor bump. Co-Authored-By: Claude Opus 4.8 (1M context) --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b1ef7807..62d82f58 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GPUCompiler" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "2.1.1" +version = "2.2.0" authors = ["Tim Besard "] [workspace] From 36d95ce3f03c83aa387497438443058d47f9e863 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 13:16:50 +0200 Subject: [PATCH 38/45] Test tweaks. --- test/native.jl | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/test/native.jl b/test/native.jl index 4e7caf20..71677168 100644 --- a/test/native.jl +++ b/test/native.jl @@ -767,7 +767,10 @@ end bytes = Vector{UInt8}(codeunits(obj)) entry = LLVM.name(meta.entry) - expected = GPUCompiler.resolve_relocation_target(only(values(relocs.sites))) + probe_site = findfirst(ref -> ref isa GPUCompiler.JuliaValueRef && + ref.value === :relocation_probe, relocs.sites) + @test probe_site !== nothing + expected = GPUCompiler.resolve_relocation_target(relocs.sites[probe_site]) fptr, keepalive = Native.load(bytes, entry, relocs, meta.ir) try GC.@preserve keepalive begin @@ -825,6 +828,11 @@ end @test isempty(meta.relocations.sites) # nothing is left for a loader to patch or import @test !any(GPUCompiler.isextinit, globals(ir)) + + # back-ends like Metal.jl drive object emission themselves, without threading + # relocation metadata through; that only works because of the baking above. + code, _ = GPUCompiler.emit_asm(job, ir, LLVM.API.LLVMObjectFile) + @test !isempty(code) end end @@ -853,7 +861,10 @@ end bytes = Vector{UInt8}(codeunits(obj)) entry = LLVM.name(meta.entry) - expected = GPUCompiler.resolve_relocation_target(only(values(relocs.sites))) + probe_site = findfirst(ref -> ref isa GPUCompiler.JuliaValueRef && + ref.value === :patch_probe, relocs.sites) + @test probe_site !== nothing + expected = GPUCompiler.resolve_relocation_target(relocs.sites[probe_site]) fptr, keepalive = Native.load(bytes, entry, relocs, meta.ir) try GC.@preserve keepalive begin From 8c6925e0228eb16a99bab65393e187f2e95650a0 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 14:03:25 +0200 Subject: [PATCH 39/45] Defer and new helper. --- src/driver.jl | 7 ++++--- src/interface.jl | 17 ++++++++++++----- src/relocation.jl | 40 +++++++++++++++++++++++++++++++++++----- test/helpers/native.jl | 15 +++++++++------ test/native.jl | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 94 insertions(+), 19 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index 91644a1d..c8e4e049 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -53,9 +53,10 @@ Compile a `job` to one of the following formats as specified by the `target` arg For the default `:bake` [`relocation_lowering`](@ref) strategy the `:llvm` result is execution-ready: Julia-value references are resolved into the IR during `emit_llvm` (it may -still contain raw `jl_*` runtime references, collected and resolved at object emission). A -back-end that defers relocation lowering keeps those references symbolic in the `:llvm` -result and returns final `relocations` metadata only for `:asm`/`:obj`. +still contain raw `jl_*` runtime references, collected and resolved at object emission). +Non-baking back-ends keep those references symbolic in the `:llvm` result: `:patch` and +`:import` return final `relocations` metadata for `:asm`/`:obj`, while `:defer` consumers +stop at `:llvm` and resolve the sites themselves with [`apply_relocations!`](@ref). """ function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing diff --git a/src/interface.jl b/src/interface.jl index 0feafd17..f84d2f85 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -316,11 +316,18 @@ dump_native(@nospecialize(job::CompilerJob)) = false # to resolve at link time (e.g. ORC `absoluteSymbols`); interior definition sites are # still patched after loading. # +# - `:defer`: keep references symbolic; the consumer takes the `:llvm` result (typically +# caching its bitcode plus the `relocations` metadata) and resolves the sites itself with +# [`apply_relocations!`](@ref) before executing the code. Object emission is unsupported +# while live relocations remain. This is the strategy for session JITs that feed IR into +# a single long-lived symbol namespace (Enzyme- or AllocCheck-style), where the +# per-object namespaces required by `:patch`/`:import` do not exist. +# # The value also fixes lowering *timing*: `:bake` resolves eagerly during `emit_llvm`, -# other strategies defer to object emission. `:patch`/`:import` loaders must keep the -# `roots` returned by [`resolved_relocations`](@ref) alive while the code can run. Generated -# code is session-portable only when [`supports_relocatable_ir`](@ref) and a non-baking -# strategy is used. +# other strategies defer past optimization. `:patch`/`:import` loaders must keep the +# `roots` returned by [`resolved_relocations`](@ref) alive while the code can run (for +# `:defer`, `apply_relocations!` returns them). Generated code is session-portable only +# when [`supports_relocatable_ir`](@ref) and a non-baking strategy is used. relocation_lowering(@nospecialize(job::CompilerJob)) = :bake # the Julia module to look up target-specific runtime functions in (this includes both @@ -529,7 +536,7 @@ end # HAS_INTEGRATED_CACHE @public RelocationSite, Relocations, relocation_lowering @public bake_relocations!, emit_patchable_relocations!, emit_imported_relocations! -@public prune_dead_relocations! +@public apply_relocations!, prune_dead_relocations! @public resolved_relocations, supports_relocatable_ir @public GPUCompilerCacheToken, cache_owner, cached_results diff --git a/src/relocation.jl b/src/relocation.jl index 4086cf26..ae2a7b6d 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -17,12 +17,16 @@ # `:bake` `bake_relocations!` none; current-session words baked JIT address folding # `:patch` `emit_patchable_relocations!` patch every site after load sysimage gvars # `:import` `emit_imported_relocations!` resolve declarations at link time sysimg symbol import +# `:defer` none (`:llvm` only) `apply_relocations!` before use n/a (session JIT) # -# Back-ends may also implement custom lowerings on top of these primitives (see the native -# test helper's JIT loader). The producers are `collect_julia_value_relocations!` during IR -# generation and `collect_cglobal_relocations!` immediately before back-end lowering. Names -# are globally unique and assigned once, so linking can merge IR and metadata without -# renaming. +# `:patch` and `:import` require a symbol namespace per loaded object (a GPU module, or a +# fresh JIT dylib): site names are only unique per compilation, and shared sites like the +# runtime library's recur verbatim across modules. Consumers that feed IR into one +# long-lived JIT namespace (Enzyme- or AllocCheck-style session JITs) use `:defer` and +# resolve sites into each parsed module with `apply_relocations!`, which needs no symbols +# at all. The producers are `collect_julia_value_relocations!` during IR generation and +# `collect_cglobal_relocations!` immediately before back-end lowering. Names are globally +# unique and assigned once, so linking can merge IR and metadata without renaming. # # Generated code is portable only when `supports_relocatable_ir()` and a non-baking strategy # preserves these relocations for its loader. The default (`:bake`) resolves in-session. @@ -464,6 +468,13 @@ function lower_relocations!(@nospecialize(job::CompilerJob), mod::LLVM.Module, emit_patchable_relocations!(mod, relocs) elseif strategy === :import emit_imported_relocations!(mod, relocs) + elseif strategy === :defer + # the consumer resolves relocations itself at the `:llvm` level; emitting an + # object here would leave its sites permanently unresolved + isempty(relocs.sites) || + error("Job defers relocation lowering to its consumer, so code with live " * + "relocations cannot be emitted. Use `apply_relocations!` on the " * + "`:llvm` result, or a `:patch`/`:import` lowering strategy.") else error("Unknown relocation lowering strategy :$strategy") end @@ -546,6 +557,25 @@ function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) return end +""" + apply_relocations!(mod, relocs) -> roots + +Loader entry point for `:defer` back-ends: resolve every live site in the current Julia +process and materialize the words into `mod`, like [`bake_relocations!`](@ref), but without +consuming `relocs` — cached metadata can be applied again to a freshly parsed module in a +future session. Sites whose global was optimized away are skipped. + +Returns the Julia `roots` that must stay alive for as long as the module's code can run. +Apply once per parsed module: the baked module has no relocations left to apply. +""" +function apply_relocations!(mod::LLVM.Module, relocs::Relocations) + live = copy(relocs) + prune_dead_relocations!(mod, live) + roots = Any[ref.value for ref in values(live.sites) if ref isa JuliaValueRef] + bake_relocations!(mod, live) + return roots +end + ## introspection diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 0a54c50c..c575e79e 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -12,10 +12,11 @@ struct CompilerParams <: AbstractCompilerParams method_table jit::Bool patch::Bool + defer::Bool CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table, - jit::Bool=false, patch::Bool=false) = - new(entry_safepoint, method_table, jit, patch) + jit::Bool=false, patch::Bool=false, defer::Bool=false) = + new(entry_safepoint, method_table, jit, patch, defer) end module Runtime end @@ -26,9 +27,11 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint -# Both non-baking modes drive an ORC loader (see `load`): `jit` imports declarations at -# link time, `patch` emits patchable definitions to write after loading. Plain jobs bake. +# The object-emitting non-baking modes drive an ORC loader (see `load`): `jit` imports +# declarations at link time, `patch` emits patchable definitions to write after loading. +# `defer` stops at `:llvm` for the consumer to `apply_relocations!`. Plain jobs bake. GPUCompiler.relocation_lowering(@nospecialize(job::NativeCompilerJob)) = + job.config.params.defer ? :defer : job.config.params.patch ? :patch : job.config.params.jit ? :import : :bake @@ -48,11 +51,11 @@ end function create_job(@nospecialize(func), @nospecialize(types); entry_safepoint::Bool=false, method_table=test_method_table, - jit::Bool=false, patch::Bool=false, kwargs...) + jit::Bool=false, patch::Bool=false, defer::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = NativeCompilerTarget(;jlruntime=true) - params = CompilerParams(entry_safepoint, method_table, jit, patch) + params = CompilerParams(entry_safepoint, method_table, jit, patch, defer) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end diff --git a/test/native.jl b/test/native.jl index 71677168..41adee20 100644 --- a/test/native.jl +++ b/test/native.jl @@ -877,6 +877,40 @@ end end end +@testset "deferred relocation" begin + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + f() = UInt(pointer_from_objref(:defer_probe)) + end + job, _ = Native.create_job(mod.f, Tuple{}; defer=true) + JuliaContext() do ctx + ir, meta = GPUCompiler.compile(:llvm, job) + relocs = meta.relocations + @test !isempty(relocs.sites) + + # a deferring consumer caches the bitcode and the relocation metadata... + bitcode = let io = IOBuffer() + write(io, ir) + take!(io) + end + + # ...and every session applies the metadata to a freshly parsed module + for _ in 1:2 + session_mod = parse(LLVM.Module, MemoryBuffer(bitcode)) + roots = GPUCompiler.apply_relocations!(session_mod, relocs) + @test :defer_probe in roots + @test !isempty(relocs.sites) # the metadata is not consumed + for site in keys(relocs.sites) + @test !isdeclaration(globals(session_mod)[site.name]) + end + end + + # emitting an object while relocations are live is a consumer error + @test_throws "defers relocation lowering" GPUCompiler.compile(:obj, job) + end + end +end + @testset "relocation validation errors" begin JuliaContext() do ctx word() = GPUCompiler.relocation_word_type() From d3e91e931a6b751f5b00c4ecaad408750241dc4d Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 14:10:13 +0200 Subject: [PATCH 40/45] Reduce exports. --- src/driver.jl | 5 +++++ src/interface.jl | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/driver.jl b/src/driver.jl index c8e4e049..2d962ca0 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -396,6 +396,11 @@ const __llvm_initialized = Ref(false) end end + # Non-baking strategies hand the `:llvm` result and its metadata to a loader + # or `:defer` consumer; drop sites that optimization killed so they don't see + # (or cache) dead entries. + bake_relocations || prune_dead_relocations!(ir, relocations) + # optimization may have replaced functions, so look the entry point up again entry = functions(ir)[entry_fn] diff --git a/src/interface.jl b/src/interface.jl index f84d2f85..fdea56cf 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -534,10 +534,12 @@ end end # HAS_INTEGRATED_CACHE +# The relocation API surface: data types, the strategy trait, the capability probe, and +# one loader entry point per consumer style (`resolved_relocations` for `:patch`/`:import` +# loaders, `apply_relocations!` for `:defer` consumers). The per-strategy lowering +# primitives (`bake_relocations!` etc.) are internal, reached through the trait. @public RelocationSite, Relocations, relocation_lowering -@public bake_relocations!, emit_patchable_relocations!, emit_imported_relocations! -@public apply_relocations!, prune_dead_relocations! -@public resolved_relocations, supports_relocatable_ir +@public apply_relocations!, resolved_relocations, supports_relocatable_ir @public GPUCompilerCacheToken, cache_owner, cached_results # the method table to use From afaa26bbc82b285575983bd70261c779912b449b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 14:37:22 +0200 Subject: [PATCH 41/45] Exclude Julia 1.10 from supports_relocatable_ir(). `jl_get_llvm_gvs_globals` was backported to 1.10, so `HAS_LLVM_GVS_GLOBALS` alone reports 1.10 as relocation-capable. But 1.10's codegen still bakes host references inline (as `inttoptr` constants) in the JIT (non-imaging) mode we compile in, rather than emitting the relocatable global declarations the relocation machinery collects, so no relocation sites are ever produced. Only 1.11+ emits those declarations, so gate the predicate on the version too. Fixes the relocation testset failures in native.jl and ptx.jl on 1.10, which all run under `if supports_relocatable_ir()`. Co-Authored-By: Claude Fable 5 --- src/utils.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils.jl b/src/utils.jl index 83be6806..bce76ef0 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -423,5 +423,9 @@ end supports_relocatable_ir() = @static if VERSION >= v"1.13.0-DEV.623" true else - HAS_LLVM_GVS_GLOBALS + # `jl_get_llvm_gvs_globals` was backported to 1.10, so the symbol alone is not enough: + # 1.10's codegen still bakes host references inline (as `inttoptr` constants) in the JIT + # (non-imaging) mode we compile in, instead of emitting the relocatable global + # declarations the relocation machinery collects. Only 1.11+ emits those declarations. + VERSION >= v"1.11-" && HAS_LLVM_GVS_GLOBALS end From 6358b03a50dc9cc56ede8a54ed93f8eacdbe6aa8 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 14:48:07 +0200 Subject: [PATCH 42/45] Bake relocations inside a zeroinitializer box. An all-zero box -- a patchable header over an all-zero payload, as produced for `SVector(0f0, 0f0)` and friends -- is folded by LLVM to a `zeroinitializer`. That ConstantAggregateZero reports no operands, so `bake_relocations!` built an empty field vector and threw `BoundsError` when writing the resolved header word (JuliaGPU/oneAPI.jl "#55: invalid integers created by alloc_opt"). Rebuild the explicit per-field constants from the struct's element types when the initializer is a ConstantAggregateZero, and add a regression test. Co-Authored-By: Claude Fable 5 --- src/relocation.jl | 9 ++++++++- test/native.jl | 22 ++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/relocation.jl b/src/relocation.jl index ae2a7b6d..0f7268e1 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -501,7 +501,14 @@ function bake_relocations!(mod::LLVM.Module, relocs::Relocations) init = initializer(gv) T = value_type(init)::LLVM.StructType idx = Int(element_at(datalayout(mod), T, site.offset)) + 1 - fields = LLVM.Constant[operands(init)...] + # An all-zero box (e.g. a patchable header over a zero payload) is folded to a + # `zeroinitializer`, a `ConstantAggregateZero` that reports no operands; rebuild + # the explicit per-field constants from the struct's element types. + fields = if init isa LLVM.ConstantAggregateZero + LLVM.Constant[null(elty) for elty in elements(T)] + else + LLVM.Constant[operands(init)...] + end fields[idx] = ConstantInt(value_type(fields[idx]), resolve_relocation_target(ref)) initializer!(gv, ConstantStruct(T, fields)) linkage!(gv, LLVM.API.LLVMPrivateLinkage) diff --git a/test/native.jl b/test/native.jl index 41adee20..a24181c9 100644 --- a/test/native.jl +++ b/test/native.jl @@ -966,6 +966,28 @@ end end end +@testset "bake zeroinitializer box" begin + # An all-zero box (a patchable header over a zero payload) is folded by LLVM to a + # `zeroinitializer`, a ConstantAggregateZero that reports no operands; baking must still + # resolve its header word. Regresses JuliaGPU/oneAPI.jl's "#55: invalid integers created + # by alloc_opt", where `SVector(0f0, 0f0)` boxed a zero payload. + JuliaContext() do ctx + mod = parse(LLVM.Module, + "@zero_box = private global { i64, [8 x i8] } zeroinitializer") + gv = globals(mod)["zero_box"] + @test initializer(gv) isa LLVM.ConstantAggregateZero # the folded shape + relocs = GPUCompiler.Relocations(Dict( + GPUCompiler.RelocationSite("zero_box", 0) => GPUCompiler.JuliaValueRef(Float64))) + GPUCompiler.bake_relocations!(mod, relocs) + init = initializer(gv) + @test !(init isa LLVM.ConstantAggregateZero) # rebuilt into explicit fields + header = convert(UInt, LLVM.Constant[operands(init)...][1]) + @test header == GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(Float64)) + @test isconstant(gv) + @test isempty(relocs.sites) + end +end + @testset "cglobal relocation" begin # JIT-private symbols like `jl_get_pgcstack_resolved` (JuliaLang/julia#61527) cannot # be looked up using `jl_cglobal`, so we should only resolve bindings that are From a0261f3d32a0beded8b924d9511b7b0a83bdb94a Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 15:42:01 +0200 Subject: [PATCH 43/45] Tighten relocatable IR caching and documentation --- src/GPUCompiler.jl | 7 +-- src/deprecated.jl | 32 +----------- src/driver.jl | 43 ++++++---------- src/interface.jl | 100 ++++++++++++++++++++------------------ src/jlgen.jl | 19 ++------ src/optim.jl | 6 +-- src/relocation.jl | 67 +++++++------------------ src/rtlib.jl | 30 +++++++----- src/utils.jl | 2 +- test/helpers/native.jl | 5 +- test/helpers/ptx.jl | 3 +- test/native.jl | 21 ++++---- test/native/precompile.jl | 38 +++++++++------ test/ptx.jl | 13 ++--- 14 files changed, 154 insertions(+), 232 deletions(-) diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index c25aa021..27dc9b4d 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -85,13 +85,10 @@ include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) + empty!(session_job_results) @static if !HAS_INTEGRATED_CACHE - # session-local results keyed by CodeInstance; entries serialized during - # GPUCompiler's own precompilation can never be valid in a later session - empty!(legacy_job_results) - # ditto for the in-process CodeCaches: CIs deposited by our own precompile - # workload carry world ages from the precompilation process + # CodeInstances created by GPUCompiler's precompile workload are process-local. empty!(GLOBAL_CI_CACHES) end diff --git a/src/deprecated.jl b/src/deprecated.jl index 7af67b40..e1b80f77 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -229,35 +229,6 @@ function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) end -## 1.10 `cached_results` -# -# Session-local storage for the per-job results structs; on 1.11+ these live on the -# `CodeInstance`s of Julia's integrated cache instead (see `interface.jl`). Keep the -# same identity here by associating results with a foreign `CodeInstance`: unrelated -# world-age advances can reuse a still-valid CI, while invalidation makes the lookup -# resolve to a new CI and therefore a new results struct. -struct LegacyJobResultEntry - config::CompilerConfig - value::Any -end - -const legacy_job_results = IdDict{CodeInstance,Vector{LegacyJobResultEntry}}() -const job_results_lock = ReentrantLock() - -function job_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} - Base.@lock job_results_lock begin - entries = get!(legacy_job_results, ci) do - LegacyJobResultEntry[] - end - for entry in entries - entry.config === config && entry.value isa V && return entry.value::V - end - v = V() - push!(entries, LegacyJobResultEntry(config, v)) - return v - end -end - function job_code_instance(@nospecialize(job::CompilerJob)) cache = WorldView(get_code_cache(job), job.world, job.world) CC.get(cache, job.source, nothing) @@ -268,10 +239,9 @@ end function cached_results(::Type{V}, job::CompilerJob) where {V} ci = job_code_instance(job) ci === nothing && return nothing - return job_results(V, ci, job.config) + return session_results(V, ci, job.config) end - end # !HAS_INTEGRATED_CACHE diff --git a/src/driver.jl b/src/driver.jl index 2d962ca0..4389abac 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -51,12 +51,9 @@ export compile Compile a `job` to one of the following formats as specified by the `target` argument: `:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. -For the default `:bake` [`relocation_lowering`](@ref) strategy the `:llvm` result is -execution-ready: Julia-value references are resolved into the IR during `emit_llvm` (it may -still contain raw `jl_*` runtime references, collected and resolved at object emission). -Non-baking back-ends keep those references symbolic in the `:llvm` result: `:patch` and -`:import` return final `relocations` metadata for `:asm`/`:obj`, while `:defer` consumers -stop at `:llvm` and resolve the sites themselves with [`apply_relocations!`](@ref). +The default [`relocation_lowering`](@ref) strategy resolves Julia-value relocations in the +`:llvm` result. Other strategies retain relocation metadata for their loader; `:defer` +consumers resolve it with [`apply_relocations!`](@ref). """ function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing @@ -65,7 +62,8 @@ function compile(target::Symbol, @nospecialize(job::CompilerJob)) return compile_unhooked(target, job) end -function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) +function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob); + resolve_relocations::Bool=true) if context(; throw_error=false) === nothing error("No active LLVM context. Use `JuliaContext()` do-block syntax to create one.") end @@ -80,7 +78,7 @@ function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) ## LLVM IR - ir, ir_meta = emit_llvm(job) + ir, ir_meta = emit_llvm(job; resolve_relocations) if output == :llvm if job.config.strip @@ -176,7 +174,8 @@ end const __llvm_initialized = Ref(false) -@locked function emit_llvm(@nospecialize(job::CompilerJob)) +@locked function emit_llvm(@nospecialize(job::CompilerJob); + resolve_relocations::Bool=true) if !__llvm_initialized[] InitializeAllTargets() InitializeAllTargetInfos() @@ -340,12 +339,9 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) - # Baking back-ends (the default) resolve host references into the IR here — - # before optimization, where `relocate_gvs!` used to run — so the optimizer sees - # concrete values and the `:llvm` result is execution-ready. Other strategies - # keep relocations symbolic for their loader, lowering them at object emission. - bake_relocations = relocation_lowering(job) === :bake - if bake_relocations + # Resolve early so optimization sees concrete values. + resolve_early = resolve_relocations && relocation_lowering(job) === :bake + if resolve_early prune_dead_relocations!(ir, relocations) bake_relocations!(ir, relocations) end @@ -374,11 +370,8 @@ const __llvm_initialized = Ref(false) end end - # Runtime linking during optimization can introduce further relocations (GC - # lowering materializes runtime calls that reference Julia globals). Bake those - # too, so a baking back-end's module is fully resolved before object emission - # even when the caller drives `emit_asm` without relocation metadata (Metal.jl). - if bake_relocations && !isempty(relocations.sites) + # Runtime linking during optimization can add relocations. + if resolve_early && !isempty(relocations.sites) prune_dead_relocations!(ir, relocations) bake_relocations!(ir, relocations) end @@ -396,10 +389,8 @@ const __llvm_initialized = Ref(false) end end - # Non-baking strategies hand the `:llvm` result and its metadata to a loader - # or `:defer` consumer; drop sites that optimization killed so they don't see - # (or cache) dead entries. - bake_relocations || prune_dead_relocations!(ir, relocations) + # Do not expose dead sites to loaders. + resolve_early || prune_dead_relocations!(ir, relocations) # optimization may have replaced functions, so look the entry point up again entry = functions(ir)[entry_fn] @@ -442,9 +433,7 @@ const __llvm_initialized = Ref(false) return ir, (; entry, compiled, relocations) end -# Forwarding method for back-ends that drive emission directly (e.g. Metal.jl's -# `emit_llvm` + `emit_asm`). Correct for baking back-ends: their relocations were resolved -# during `emit_llvm`, so no metadata needs threading through here. +# Compatibility for back-ends that resolve relocations during `emit_llvm`. emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, format::LLVM.API.LLVMCodeGenFileType) = emit_asm(job, ir, Relocations(), format) diff --git a/src/interface.jl b/src/interface.jl index fdea56cf..43013f59 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -302,32 +302,20 @@ can_vectorize(@nospecialize(job::CompilerJob)) = false # Should emit PTLS lookup that can be relocated dump_native(@nospecialize(job::CompilerJob)) = false -# How this back-end lowers relocations (host references) before object emission. One of: -# -# - `:bake` (default): resolve the references in the current session and materialize them -# into the IR. The `:llvm` result is execution-ready and no loader is needed. Baked code -# embeds session-local addresses, so it must not be persisted in `cached_results`. -# -# - `:patch`: keep references symbolic through optimization and emit every site as a -# patchable (null-init, externally-initialized) definition. The loader writes each -# resolved word at `global + offset` after loading the object. -# -# - `:import`: keep references symbolic and leave declaration slots external for the loader -# to resolve at link time (e.g. ORC `absoluteSymbols`); interior definition sites are -# still patched after loading. -# -# - `:defer`: keep references symbolic; the consumer takes the `:llvm` result (typically -# caching its bitcode plus the `relocations` metadata) and resolves the sites itself with -# [`apply_relocations!`](@ref) before executing the code. Object emission is unsupported -# while live relocations remain. This is the strategy for session JITs that feed IR into -# a single long-lived symbol namespace (Enzyme- or AllocCheck-style), where the -# per-object namespaces required by `:patch`/`:import` do not exist. -# -# The value also fixes lowering *timing*: `:bake` resolves eagerly during `emit_llvm`, -# other strategies defer past optimization. `:patch`/`:import` loaders must keep the -# `roots` returned by [`resolved_relocations`](@ref) alive while the code can run (for -# `:defer`, `apply_relocations!` returns them). Generated code is session-portable only -# when [`supports_relocatable_ir`](@ref) and a non-baking strategy is used. +""" + relocation_lowering(job) -> Symbol + +Select how a back-end lowers relocations: + +- `:bake` resolves them in `emit_llvm`. +- `:patch` emits definitions for the loader to patch. +- `:import` emits declarations for the loader to resolve; interior sites still need patching. +- `:defer` leaves the `:llvm` result for [`apply_relocations!`](@ref). Object emission + requires all relocations to have been applied. + +Relocatable output requires [`supports_relocatable_ir`](@ref). Loaders must retain the roots +returned by [`resolved_relocations`](@ref) or [`apply_relocations!`](@ref). +""" relocation_lowering(@nospecialize(job::CompilerJob)) = :bake # the Julia module to look up target-specific runtime functions in (this includes both @@ -404,6 +392,33 @@ else cache_owner(job.config.target, job.config.params, job.config.always_inline) end +struct SessionJobResultEntry + config::CompilerConfig + value::Any +end + +const session_job_results = IdDict{CodeInstance,Vector{SessionJobResultEntry}}() +const session_job_results_lock = ReentrantLock() + +function session_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} + Base.@lock session_job_results_lock begin + entries = get!(session_job_results, ci) do + SessionJobResultEntry[] + end + for entry in entries + entry.config === config && entry.value isa V && return entry.value::V + end + value = V() + push!(entries, SessionJobResultEntry(config, value)) + return value + end +end + +function can_persist_results(@nospecialize(job::CompilerJob)) + supports_relocatable_ir() && + relocation_lowering(job) in (:patch, :import, :defer) +end + """ cached_results(::Type{V}, job::CompilerJob) -> Union{Nothing,V} @@ -435,21 +450,13 @@ post-compile lookup in the example is guaranteed to succeed. To attach results w generating code — e.g. from an inference-only precompilation workload — run `precompile(job)` first. -Results are keyed by the *full* compiler job: its method instance, world age, and entire -`CompilerConfig` (two jobs differing only in, say, the kernel `name` get distinct -structs). Storage differs per Julia version, transparently to the back-end: +Results are keyed by the method instance, world age, and full `CompilerConfig`. On Julia +1.11+, results persist with the `CodeInstance` only when the back-end selects `:patch`, +`:import`, or `:defer` and [`supports_relocatable_ir`](@ref) is true. Other results use a +session-local store. Julia 1.10 always uses the session-local store. -- On Julia 1.11+, the struct lives on the `CodeInstance`'s `analysis_results` chain in - Julia's integrated code cache, partitioned by [`cache_owner`](@ref) and keyed by - config within the per-CI [`JobResults`](@ref) container. Method redefinition - invalidates the CI — and with it the attached results. Artifacts stored during - precompilation are serialized into the package image along with the CI: populate only - session-portable values (bytes, strings) during precompile workloads, and keep - session-local handles (device modules, pipeline objects) in fields that remain empty - until first use at run time. - -- On Julia 1.10, the struct lives in a session-local store keyed by the foreign - `CodeInstance` and config. Nothing persists across sessions. +Persistent result structs may contain session-local handles, but back-ends must leave those +fields empty during precompilation. Thread safety: concurrent calls for the same job return the same struct, but GPUCompiler does not serialize back-end *compilation*; take a back-end lock around @@ -489,7 +496,7 @@ const cached_results_lock = ReentrantLock() # NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of # instantiations: one per back-end and results type). -function job_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} +function persistent_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} jr = CompilerCaching.results(JobResults, ci) Base.@lock cached_results_lock begin for entry in jr.entries @@ -529,15 +536,16 @@ end function cached_results(::Type{V}, job::CompilerJob) where {V} ci = job_code_instance(job) ci === nothing && return nothing - return job_results(V, ci, job.config) + if can_persist_results(job) + return persistent_results(V, ci, job.config) + else + return session_results(V, ci, job.config) + end end end # HAS_INTEGRATED_CACHE -# The relocation API surface: data types, the strategy trait, the capability probe, and -# one loader entry point per consumer style (`resolved_relocations` for `:patch`/`:import` -# loaders, `apply_relocations!` for `:defer` consumers). The per-strategy lowering -# primitives (`bake_relocations!` etc.) are internal, reached through the trait. +# Public relocation interface. @public RelocationSite, Relocations, relocation_lowering @public apply_relocations!, resolved_relocations, supports_relocatable_ir @public GPUCompilerCacheToken, cache_owner, cached_results diff --git a/src/jlgen.jl b/src/jlgen.jl index ad0cebf3..93691d55 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -563,14 +563,10 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end end - # Maintain a map from global variables to their initialized Julia values. The - # objects pointed to are perma-rooted during codegen. We *don't* bake these - # addresses into the IR yet, so that we can cache it across sessions. + # Map global variables to their rooted Julia values without embedding addresses in IR. gv_to_value = Dict{String, Ptr{Cvoid}}() if gvs === nothing - # No reliable GV table on this Julia — best-effort discovery from the module. - # On these older versions Julia's own emit may have already baked in absolute - # pointer values; we recover them by reading existing initializers. + # Without a reliable GV table, recover addresses from existing initializers. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -597,15 +593,8 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init end - # Strip the initializers so the IR we hand back is session-portable - # (on 1.12 `jl_emit_native_impl` bakes pointers via - # `literal_static_pointer_val`; on 1.13+ Julia nulls them itself); - # relocation lowering writes the session-current value in. Demote - # each GV to an external *declaration* rather than - # giving it a null initializer: an internal global initialized to null - # is fair game for optimization passes that run before relocation - # (e.g. GlobalOpt in the PTX back-end's `finish_module!`), which would - # fold loads of it to null and delete the global. + # Discard session addresses. Declarations survive optimization until relocation + # lowering; null definitions could be folded away first. for gv in globals(llvm_mod) haskey(gv_to_value, LLVM.name(gv)) || continue initializer!(gv, nothing) diff --git a/src/optim.jl b/src/optim.jl index 8522b707..932f316b 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -326,11 +326,7 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op add!(fpm, GPULowerGCFramePass(job)) end if job.config.libraries - # Add GPULinkRuntime *by name* so the pass builder resolves it to the instance - # registered in `optimize!`: that instance holds the `Relocations` object owned - # by `optimize!`, and relinking the runtime here must merge new sites into it. - # The other two passes are stateless with respect to that object, so fresh - # job-capturing instances are equivalent. + # Use the registered pass because it owns `relocs`; the others only capture `job`. add!(mpm, "GPULinkRuntime") add!(mpm, GPULinkLibrariesPass(job)) add!(mpm, GPUFinishRuntimeIntrinsicsPass(job)) diff --git a/src/relocation.jl b/src/relocation.jl index 0f7268e1..f259f8c6 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -1,35 +1,11 @@ -# Relocation model -# -# A relocation is a site => target pair. Targets are Julia object identities -# (`JuliaValueRef`, like codegen's identity-keyed `global_targets`) or named -# libjulia globals (`CGlobalRef`, like JuliaVariables). Every site names a word -# at a byte offset in a global. Dedicated sites use offset zero in a word-sized -# declaration (like `jl_sysimg_gvars`); other sites are inside defined globals -# (like `gctags_list`/`relocs_list`). A declaration can be baked, patched, or -# resolved by a loader; a word inside a definition can only be baked or patched. +# Relocations map words in globals to Julia values or named C globals. Declarations can be +# resolved, patched, or imported; sites inside definitions can only be resolved or patched. # # produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower # -# The back-end's `relocation_lowering` strategy picks how (and when) sites lower: -# -# Strategy Lowering Loader contract Julia analog -# -------- -------- --------------- ------------ -# `:bake` `bake_relocations!` none; current-session words baked JIT address folding -# `:patch` `emit_patchable_relocations!` patch every site after load sysimage gvars -# `:import` `emit_imported_relocations!` resolve declarations at link time sysimg symbol import -# `:defer` none (`:llvm` only) `apply_relocations!` before use n/a (session JIT) -# -# `:patch` and `:import` require a symbol namespace per loaded object (a GPU module, or a -# fresh JIT dylib): site names are only unique per compilation, and shared sites like the -# runtime library's recur verbatim across modules. Consumers that feed IR into one -# long-lived JIT namespace (Enzyme- or AllocCheck-style session JITs) use `:defer` and -# resolve sites into each parsed module with `apply_relocations!`, which needs no symbols -# at all. The producers are `collect_julia_value_relocations!` during IR generation and -# `collect_cglobal_relocations!` immediately before back-end lowering. Names are globally -# unique and assigned once, so linking can merge IR and metadata without renaming. -# -# Generated code is portable only when `supports_relocatable_ir()` and a non-baking strategy -# preserves these relocations for its loader. The default (`:bake`) resolves in-session. +# `:patch` and `:import` need a symbol namespace per object. `:defer` instead applies sites +# directly to each parsed module. Site names are fixed at creation so IR and metadata can be +# linked without renaming. ## targets @@ -55,11 +31,9 @@ end """ CGlobalRef(symbol, library=nothing) -A named C data global. With `library === nothing` (the default) it names a libjulia global, -resolved with `jl_cglobal`'s process-wide symbol lookup; an explicit `library` names a -shared object to load and look the symbol up in. Resolution returns the word stored in that -global in the current Julia process. Both fields are plain data, so serialized metadata -stays portable. +A named C data global. With `library === nothing`, resolution uses `jl_cglobal`'s +process-wide lookup. Otherwise it looks up `symbol` in `library`. Resolution returns the +word stored in the global. """ struct CGlobalRef symbol::Symbol @@ -86,10 +60,7 @@ same_relocation_target(::RelocationTarget, ::RelocationTarget) = false Resolve a relocation target to its word in the current Julia process. """ function resolve_relocation_target(target::JuliaValueRef) - box = Any[target.value] - GC.@preserve box begin - return unsafe_load(Base.unsafe_convert(Ptr{UInt}, pointer(box))) - end + UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), target.value)) end function resolve_relocation_target(target::CGlobalRef) if target.library === nothing @@ -135,9 +106,7 @@ end Relocations() = Relocations(Dict{RelocationSite,RelocationTarget}()) -# A loader that bakes from cached metadata consumes the sites (baking empties them), so it -# needs a fresh copy per link. The site keys and targets are immutable, so a shallow copy of -# the dict suffices. +# Resolving into IR consumes the site table; loaders copy cached metadata first. Base.copy(relocs::Relocations) = Relocations(copy(relocs.sites)) """ @@ -484,10 +453,9 @@ end """ bake_relocations!(mod, relocs) -Resolve every site in the current Julia process and materialize the resulting words into -the IR, leaving `relocs` empty. The module is execution-ready but embeds session-local -addresses, so it must not be persisted across sessions. Dead sites must be dropped first -with [`prune_dead_relocations!`](@ref); baking errors on a site whose global is gone. +Resolve every site in the current Julia process and write the resulting words into the IR, +leaving `relocs` empty. The module then embeds session-local addresses and must not be +persisted across sessions. Drop dead sites first with [`prune_dead_relocations!`](@ref). """ function bake_relocations!(mod::LLVM.Module, relocs::Relocations) foreach_relocation(mod, relocs) do site, gv, ref @@ -567,13 +535,12 @@ end """ apply_relocations!(mod, relocs) -> roots -Loader entry point for `:defer` back-ends: resolve every live site in the current Julia -process and materialize the words into `mod`, like [`bake_relocations!`](@ref), but without -consuming `relocs` — cached metadata can be applied again to a freshly parsed module in a -future session. Sites whose global was optimized away are skipped. +Loader entry point for `:defer` back-ends. Resolves every live site into `mod` without +consuming `relocs`, so cached metadata can be reused. Sites whose global was optimized away +are skipped. Returns the Julia `roots` that must stay alive for as long as the module's code can run. -Apply once per parsed module: the baked module has no relocations left to apply. +Apply once per parsed module. """ function apply_relocations!(mod::LLVM.Module, relocs::Relocations) live = copy(relocs) diff --git a/src/rtlib.jl b/src/rtlib.jl index 8e619c4e..d7bdd87b 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,12 +60,9 @@ end ## functionality to build the runtime library -# Per-function compilation results for the GPU runtime library, cached through the -# same `cached_results` mechanism back-ends use for kernels. On 1.11+ the bitcode -# thus lives on the runtime function's `CodeInstance` — possibly alongside a -# back-end's own results struct — and persists through precompilation, so sessions -# loading a back-end that compiled its runtime during precompile skip codegen -# entirely. On 1.10 it is cached for the duration of the session. +# Per-function relocatable bitcode for the GPU runtime library. When Julia exposes the +# required relocation metadata, this persists with the function's `CodeInstance`. +# `runtime_libs` below provides the session cache on older Julia versions. mutable struct RuntimeFunctionResults bitcode::Union{Nothing,Vector{UInt8}} relocations::Relocations @@ -91,7 +88,9 @@ function emit_function!(mod, relocs::Relocations, config::CompilerConfig, return ci::CodeInstance end - new_mod, meta = compile_unhooked(:llvm, rt_job) + # Keep this intermediate module relocatable even when the final back-end resolves + # relocations eagerly. The caller links a fresh copy and lowers the merged sites. + new_mod, meta = compile_unhooked(:llvm, rt_job; resolve_relocations=false) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) if return_type(ft) != return_type(expected_ft) @@ -116,7 +115,7 @@ function emit_function!(mod, relocs::Relocations, config::CompilerConfig, write(io, new_mod) ci === nothing && (ci = runtime_code_instance(rt_job)) if supports_relocatable_ir() - res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) + res === nothing && (res = runtime_results(RuntimeFunctionResults, ci, rt_job.config)) res.bitcode = take!(io) res.relocations = meta.relocations end @@ -125,10 +124,19 @@ function emit_function!(mod, relocs::Relocations, config::CompilerConfig, return ci::CodeInstance end +function runtime_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} + @static if HAS_INTEGRATED_CACHE + if supports_relocatable_ir() + return persistent_results(V, ci, config) + end + end + return session_results(V, ci, config) +end + function runtime_function_results(@nospecialize(job::CompilerJob)) ci = job_code_instance(job) ci === nothing && return nothing, nothing - return ci, job_results(RuntimeFunctionResults, ci, job.config) + return ci, runtime_results(RuntimeFunctionResults, ci, job.config) end function runtime_method_instance(@nospecialize(job::CompilerJob), method) @@ -190,8 +198,8 @@ end # Session-local cache of assembled runtime libraries, keyed by # `(runtime_config(job), opaque_pointers)`: the derived runtime config covers every -# codegen-relevant setting (e.g. the debug level, which is baked into the runtime IR -# as a constant), while cosmetic kernel-job fields are normalized away. Cross-session +# codegen-relevant setting (e.g. the debug level stored in the runtime IR), while +# cosmetic kernel-job fields are normalized away. Cross-session # persistence happens at the per-function level (see `RuntimeFunctionResults`): # reassemble on first use of each session, then reuse within the session. # diff --git a/src/utils.jl b/src/utils.jl index bce76ef0..cae25a8e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -424,7 +424,7 @@ supports_relocatable_ir() = @static if VERSION >= v"1.13.0-DEV.623" true else # `jl_get_llvm_gvs_globals` was backported to 1.10, so the symbol alone is not enough: - # 1.10's codegen still bakes host references inline (as `inttoptr` constants) in the JIT + # 1.10's codegen still embeds Julia addresses (as `inttoptr` constants) in the JIT # (non-imaging) mode we compile in, instead of emitting the relocatable global # declarations the relocation machinery collects. Only 1.11+ emits those declarations. VERSION >= v"1.11-" && HAS_LLVM_GVS_GLOBALS diff --git a/test/helpers/native.jl b/test/helpers/native.jl index c575e79e..7a1a3408 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -27,9 +27,8 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint -# The object-emitting non-baking modes drive an ORC loader (see `load`): `jit` imports -# declarations at link time, `patch` emits patchable definitions to write after loading. -# `defer` stops at `:llvm` for the consumer to `apply_relocations!`. Plain jobs bake. +# Object-emitting modes drive an ORC loader (see `load`): `jit` imports declarations, +# while `patch` emits definitions to write after loading. `defer` stops at `:llvm`. GPUCompiler.relocation_lowering(@nospecialize(job::NativeCompilerJob)) = job.config.params.defer ? :defer : job.config.params.patch ? :patch : diff --git a/test/helpers/ptx.jl b/test/helpers/ptx.jl index 1bf2cbeb..d2233325 100644 --- a/test/helpers/ptx.jl +++ b/test/helpers/ptx.jl @@ -10,8 +10,7 @@ end PTXCompilerJob = CompilerJob{PTXCompilerTarget,CompilerParams} -# `patch=true` keeps relocations symbolic (as CUDA.jl does) so tests can observe them -# surviving to the `:llvm`/`:asm` output; otherwise bake in-session like a plain compilation. +# `patch=true` keeps relocations symbolic (as CUDA.jl does); plain jobs resolve them in IR. GPUCompiler.relocation_lowering(@nospecialize(job::PTXCompilerJob)) = job.config.params.patch ? :patch : :bake diff --git a/test/native.jl b/test/native.jl index a24181c9..6156d1f2 100644 --- a/test/native.jl +++ b/test/native.jl @@ -59,7 +59,7 @@ end end end - # a non-baking (JIT) back-end keeps the Symbol reference symbolic in `:llvm`. + # A JIT back-end keeps the Symbol reference symbolic in `:llvm`. job, _ = Native.create_job(mod.outer, (Int, Symbol); validate=false, jit=true) JuliaContext() do ctx ir, meta = GPUCompiler.compile(:llvm, job) @@ -274,7 +274,7 @@ end @test haskey(lib.relocations.sites, site) end if GPUCompiler.supports_relocatable_ir() - # otherwise Julia bakes addresses without tagging globals + # otherwise Julia embeds addresses without tagging globals @test used > 0 end end @@ -416,7 +416,7 @@ end GPUCompiler.resolve_relocation_target(ref) dispose(m) - # Symbol: baked address + # Symbol: resolved address m, map = slot_module(ptrs[3]) relocs = GPUCompiler.collect_julia_value_relocations!(m, map) @test only(values(relocs.sites)).value === objs[3] @@ -815,10 +815,8 @@ end end end -@testset "eager relocation baking" begin - # The default (baking) back-end resolves host references into the IR during `emit_llvm`, - # so the `:llvm` result carries no relocation metadata and needs no loader — the contract - # that lets Metal.jl and unmodified AllocCheck consume it directly. +@testset "eager relocation resolution" begin + # Eager resolution in `emit_llvm` leaves nothing for a loader. mod = @eval module $(gensym()) probe() = UInt(pointer_from_objref(:eager_probe)) end @@ -829,8 +827,7 @@ end # nothing is left for a loader to patch or import @test !any(GPUCompiler.isextinit, globals(ir)) - # back-ends like Metal.jl drive object emission themselves, without threading - # relocation metadata through; that only works because of the baking above. + # This back-end can emit objects without threading relocation metadata. code, _ = GPUCompiler.emit_asm(job, ir, LLVM.API.LLVMObjectFile) @test !isempty(code) end @@ -966,9 +963,9 @@ end end end -@testset "bake zeroinitializer box" begin +@testset "resolve zeroinitializer box" begin # An all-zero box (a patchable header over a zero payload) is folded by LLVM to a - # `zeroinitializer`, a ConstantAggregateZero that reports no operands; baking must still + # `zeroinitializer`, a ConstantAggregateZero that reports no operands; resolution must # resolve its header word. Regresses JuliaGPU/oneAPI.jl's "#55: invalid integers created # by alloc_opt", where `SVector(0f0, 0f0)` boxed a zero payload. JuliaContext() do ctx @@ -1560,7 +1557,7 @@ end end end - # a non-baking job keeps the merged relocation symbolic so we can inspect it. + # Keep the merged relocation symbolic so we can inspect it. job, _ = Native.create_job(mod.parent, (Symbol,); jit=true, validate=false) JuliaContext() do ctx ir, meta = GPUCompiler.compile(:llvm, job) diff --git a/test/native/precompile.jl b/test/native/precompile.jl index cd257142..282f4f6e 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -37,20 +37,21 @@ precompile_test_harness("Inference caching") do load_path Results() = new(nothing) end - portable_kernel(x) = x + 1 - cached_kernel(x) = x + 2 + persistent_kernel(x) = x + 1 + session_kernel(x) = x + 2 - # Job results are serialized as-is. Backends are responsible for storing only - # artifacts allowed by the static relocatability contract. + # A back-end implementing relocation keeps results across sessions. let - job, _ = NativeCompiler.Native.create_job(portable_kernel, (Int,)) + job, _ = NativeCompiler.Native.create_job(persistent_kernel, (Int,); jit=true) precompile(job) - NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "portable" + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "persistent" end + + # Default back-ends use the session-local store. let - job, _ = NativeCompiler.Native.create_job(cached_kernel, (Int,)) + job, _ = NativeCompiler.Native.create_job(session_kernel, (Int,)) precompile(job) - NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "cached" + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "session" end let @@ -109,15 +110,20 @@ precompile_test_harness("Inference caching") do load_path using NativeBackend - portable_job, _ = NativeCompiler.Native.create_job(NativeBackend.portable_kernel, (Int,)) - portable_res = GPUCompiler.cached_results(NativeBackend.Results, portable_job) - @test portable_res !== nothing - @test portable_res.artifact == "portable" + persistent_job, _ = NativeCompiler.Native.create_job( + NativeBackend.persistent_kernel, (Int,); jit=true) + persistent_res = GPUCompiler.cached_results(NativeBackend.Results, persistent_job) + @test persistent_res !== nothing + if GPUCompiler.supports_relocatable_ir() + @test persistent_res.artifact == "persistent" + else + @test persistent_res.artifact === nothing + end - cached_job, _ = NativeCompiler.Native.create_job(NativeBackend.cached_kernel, (Int,)) - cached_res = GPUCompiler.cached_results(NativeBackend.Results, cached_job) - @test cached_res !== nothing - @test cached_res.artifact == "cached" + session_job, _ = NativeCompiler.Native.create_job(NativeBackend.session_kernel, (Int,)) + session_res = GPUCompiler.cached_results(NativeBackend.Results, session_job) + @test session_res !== nothing + @test session_res.artifact === nothing # Check that kernel survived kernel_mi = GPUCompiler.methodinstance(typeof(NativeBackend.kernel), Tuple{Vector{Int}, Int}) diff --git a/test/ptx.jl b/test/ptx.jl index de773404..661d0f89 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -29,19 +29,16 @@ end end @testset "global variable relocation" begin - # references to Julia objects (`julia.constgv` globals, e.g. Symbol literals) must - # survive as relocation slots until backend lowering at object emission. - # they used to be kept alive as internal globals with a null initializer, which the - # GlobalOpt run in `finish_module!` folded away, constant-folding any comparison - # against them (JuliaGPU/CUDA.jl#3185: kernels specialized on Symbols misbehaved). + # Julia-object references must remain declarations until relocation lowering. Null + # definitions would let GlobalOpt fold comparisons against them. mod = @eval module $(gensym()) kernel(name::Symbol) = name === :var ? 1 : 2 end - # `patch=true` keeps the slot symbolic (a baking back-end would resolve and fold it away). + # `patch=true` keeps the slot symbolic instead of resolving it into the IR. ir = sprint() do io PTX.code_llvm(io, mod.kernel, Tuple{Symbol}; dump_module=true, patch=true) end - # Julia bakes the Symbol pointer into the IR before exposing it to GPUCompiler when it + # Julia embeds the Symbol pointer before exposing the IR to GPUCompiler when it # can't emit relocatable global metadata; otherwise it stays a symbolic slot we preserve. if GPUCompiler.supports_relocatable_ir() @test occursin("@jl_sym_var_", ir) @@ -102,7 +99,7 @@ end # `patch=true` keeps the interior header relocation symbolic (externally_initialized). ir = sprint(io->PTX.code_llvm(io, mod.consume, Tuple{Bool,Int32}; dump_module=true, patch=true)) - # As with Symbol literals above, Julia bakes the type pointer when it can't emit + # As with Symbol literals above, Julia embeds the type pointer when it can't emit # relocatable global metadata; otherwise GPUCompiler represents it as a relocation. if GPUCompiler.supports_relocatable_ir() @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) From 44c1e41259d5f7915f613a60188b84098eda0817 Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 15:55:34 +0200 Subject: [PATCH 44/45] Clarify result cache naming --- src/GPUCompiler.jl | 2 +- src/interface.jl | 36 ++++++++++++++++++------------------ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index 27dc9b4d..98bb256d 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -85,7 +85,7 @@ include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) - empty!(session_job_results) + empty!(session_results_cache) @static if !HAS_INTEGRATED_CACHE # CodeInstances created by GPUCompiler's precompile workload are process-local. diff --git a/src/interface.jl b/src/interface.jl index 43013f59..3a8a3da0 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -392,24 +392,24 @@ else cache_owner(job.config.target, job.config.params, job.config.always_inline) end -struct SessionJobResultEntry +struct SessionResultEntry config::CompilerConfig value::Any end -const session_job_results = IdDict{CodeInstance,Vector{SessionJobResultEntry}}() -const session_job_results_lock = ReentrantLock() +const session_results_cache = IdDict{CodeInstance,Vector{SessionResultEntry}}() +const session_results_lock = ReentrantLock() function session_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} - Base.@lock session_job_results_lock begin - entries = get!(session_job_results, ci) do - SessionJobResultEntry[] + Base.@lock session_results_lock begin + entries = get!(session_results_cache, ci) do + SessionResultEntry[] end for entry in entries entry.config === config && entry.value isa V && return entry.value::V end value = V() - push!(entries, SessionJobResultEntry(config, value)) + push!(entries, SessionResultEntry(config, value)) return value end end @@ -467,7 +467,7 @@ function cached_results end @static if HAS_INTEGRATED_CACHE """ - JobResults + PersistentJobResults Per-CodeInstance container mapping a `CompilerConfig` to the back-end's results struct. @@ -482,28 +482,28 @@ from package images. # `CompilerConfig` is an abstract UnionAll here. Keeping it in a tuple stored inline in a # Vector boxes the config again on every iteration. A non-isbits entry object is stored by # reference, so both fields are boxed once and hot-path scans allocate nothing. -struct JobResultEntry +struct PersistentResultEntry config::CompilerConfig value::Any end -mutable struct JobResults - entries::Vector{JobResultEntry} - JobResults() = new(JobResultEntry[]) +mutable struct PersistentJobResults + entries::Vector{PersistentResultEntry} + PersistentJobResults() = new(PersistentResultEntry[]) end -const cached_results_lock = ReentrantLock() +const persistent_results_lock = ReentrantLock() # NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of # instantiations: one per back-end and results type). function persistent_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} - jr = CompilerCaching.results(JobResults, ci) - Base.@lock cached_results_lock begin - for entry in jr.entries + results = CompilerCaching.results(PersistentJobResults, ci) + Base.@lock persistent_results_lock begin + for entry in results.entries entry.config === config && entry.value isa V && return entry.value::V end v = V() - push!(jr.entries, JobResultEntry(config, v)) + push!(results.entries, PersistentResultEntry(config, v)) return v end end @@ -511,7 +511,7 @@ end function cache_view(@nospecialize(job::CompilerJob)) # `cache_owner` is deliberately stored as Any on CompilerConfig: preserve that box in the # CacheView instead of re-specializing and re-boxing the immutable token for the ccall. - CompilerCaching.CacheView{Any,JobResults}(cache_owner(job), job.world) + CompilerCaching.CacheView{Any,PersistentJobResults}(cache_owner(job), job.world) end # Fetch the CodeInstance backing `job` from the integrated cache. On 1.14+, inference From 86d4dac86ba455ae7c27a149d61608652263744b Mon Sep 17 00:00:00 2001 From: Tim Besard Date: Thu, 23 Jul 2026 20:09:43 +0200 Subject: [PATCH 45/45] Permanently root resolved relocation targets. Resolving a JuliaValueRef now routes through jl_as_global_root, exactly as Julia's own codegen roots values referenced from native code (jl_ensure_rooted on 1.10/1.11, aot_optimize_roots on 1.12+). Values compiled in the active session are already rooted that way, making this a cheap lookup; it matters for relocation metadata deserialized from a cache, whose values codegen never saw. Rooting by egal identity also folds such duplicates onto the instance native code already uses. With addresses guaranteed for the lifetime of the session, loaders no longer need GC bookkeeping, so resolved_relocations and apply_relocations! stop returning roots. Co-Authored-By: Claude Fable 5 --- src/interface.jl | 5 +++-- src/relocation.jl | 46 +++++++++++++++++++++++++++--------------- test/helpers/native.jl | 6 +++--- test/native.jl | 3 +-- 4 files changed, 37 insertions(+), 23 deletions(-) diff --git a/src/interface.jl b/src/interface.jl index 3a8a3da0..22ba6622 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -313,8 +313,9 @@ Select how a back-end lowers relocations: - `:defer` leaves the `:llvm` result for [`apply_relocations!`](@ref). Object emission requires all relocations to have been applied. -Relocatable output requires [`supports_relocatable_ir`](@ref). Loaders must retain the roots -returned by [`resolved_relocations`](@ref) or [`apply_relocations!`](@ref). +Relocatable output requires [`supports_relocatable_ir`](@ref). Resolving relocations +permanently roots the referenced Julia values in the process (mirroring Julia's own +codegen), so loaders need no GC bookkeeping of their own. """ relocation_lowering(@nospecialize(job::CompilerJob)) = :bake diff --git a/src/relocation.jl b/src/relocation.jl index f259f8c6..30fc65ab 100644 --- a/src/relocation.jl +++ b/src/relocation.jl @@ -15,8 +15,8 @@ A Julia value with a stable address (a heap object, symbol, or singleton), used as the serializable identity of a relocation target. Resolve it in the active session with -[`resolve_relocation_target`](@ref); a loader that writes the resulting address into device -storage must keep `value` rooted for as long as that storage remains reachable. +[`resolve_relocation_target`](@ref), which permanently roots the value in the process so +the resolved address stays valid for as long as the session lives. """ struct JuliaValueRef value::Any @@ -54,13 +54,31 @@ same_relocation_target(a::CGlobalRef, b::CGlobalRef) = a.symbol === b.symbol && a.library == b.library same_relocation_target(::RelocationTarget, ::RelocationTarget) = false +# Permanently root a value in the current process and return the canonical rooted +# instance, exactly as Julia's own codegen does for values referenced from native code +# (`jl_ensure_rooted` on 1.10/1.11, `aot_optimize_roots` on 1.12+). Values compiled in +# this session are already rooted this way, making this a cheap lookup; it matters for +# metadata deserialized from a cache, whose values codegen never saw. Rooting by egal +# identity also folds such duplicates onto the instance native code already uses. +function root_relocation_target(target::JuliaValueRef) + @static if VERSION >= v"1.11-" + ccall(:jl_as_global_root, Any, (Any, Cint), target.value, 1) + else + ccall(:jl_as_global_root, Any, (Any,), target.value) + end +end + +value_pointer(@nospecialize(value)) = UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), value)) + """ resolve_relocation_target(target) -> UInt -Resolve a relocation target to its word in the current Julia process. +Resolve a relocation target to its word in the current Julia process. Julia values are +permanently rooted (and canonicalized by egal identity) as part of resolution, so the +returned address cannot dangle. """ function resolve_relocation_target(target::JuliaValueRef) - UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), target.value)) + value_pointer(root_relocation_target(target)) end function resolve_relocation_target(target::CGlobalRef) if target.library === nothing @@ -110,19 +128,18 @@ Relocations() = Relocations(Dict{RelocationSite,RelocationTarget}()) Base.copy(relocs::Relocations) = Relocations(copy(relocs.sites)) """ - resolved_relocations(relocs) + resolved_relocations(relocs) -> Vector{Pair{RelocationSite,UInt}} -Resolve relocation metadata for a loader. Returns resolved `sites` and Julia `roots` that -must stay alive while the loaded code can access them. +Resolve relocation metadata for a loader, returning each site with its resolved word. +Resolution permanently roots referenced Julia values in the process, so the addresses +stay valid for the lifetime of the session. """ function resolved_relocations(relocs::Relocations) sites = Pair{RelocationSite,UInt}[] - roots = Any[] for (site, ref) in relocs.sites push!(sites, site => resolve_relocation_target(ref)) - ref isa JuliaValueRef && push!(roots, ref.value) end - return (; sites, roots) + return sites end relocation_word_type() = LLVM.IntType(8sizeof(UInt)) @@ -533,21 +550,18 @@ function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) end """ - apply_relocations!(mod, relocs) -> roots + apply_relocations!(mod, relocs) Loader entry point for `:defer` back-ends. Resolves every live site into `mod` without consuming `relocs`, so cached metadata can be reused. Sites whose global was optimized away -are skipped. - -Returns the Julia `roots` that must stay alive for as long as the module's code can run. +are skipped. Resolution permanently roots referenced Julia values in the process. Apply once per parsed module. """ function apply_relocations!(mod::LLVM.Module, relocs::Relocations) live = copy(relocs) prune_dead_relocations!(mod, live) - roots = Any[ref.value for ref in values(live.sites) if ref isa JuliaValueRef] bake_relocations!(mod, live) - return roots + return end diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 7a1a3408..e8fa597d 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -68,7 +68,7 @@ function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations add!(jd, LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)) relocations = GPUCompiler.resolved_relocations(relocs) - declarations = [(site, value) for (site, value) in relocations.sites + declarations = [(site, value) for (site, value) in relocations if isdeclaration(globals(ir)[site.name])] cells = Vector{UInt}(undef, length(declarations)) pairs = LLVM.API.LLVMOrcCSymbolMapPair[] @@ -83,13 +83,13 @@ function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations isempty(pairs) || LLVM.define(jd, LLVM.absolute_symbols(pairs)) add!(lljit, jd, MemoryBuffer(obj)) - for (site, value) in relocations.sites + for (site, value) in relocations isdeclaration(globals(ir)[site.name]) && continue addr = lookup(lljit, site.name) unsafe_store!(Ptr{UInt}(pointer(addr) + site.offset), value) end addr = lookup(lljit, entry) - return pointer(addr), (lljit, cells, relocations.roots) + return pointer(addr), (lljit, cells) catch dispose(lljit) rethrow() diff --git a/test/native.jl b/test/native.jl index 6156d1f2..2b50a2cd 100644 --- a/test/native.jl +++ b/test/native.jl @@ -894,8 +894,7 @@ end # ...and every session applies the metadata to a freshly parsed module for _ in 1:2 session_mod = parse(LLVM.Module, MemoryBuffer(bitcode)) - roots = GPUCompiler.apply_relocations!(session_mod, relocs) - @test :defer_probe in roots + GPUCompiler.apply_relocations!(session_mod, relocs) @test !isempty(relocs.sites) # the metadata is not consumed for site in keys(relocs.sites) @test !isdeclaration(globals(session_mod)[site.name])