diff --git a/Project.toml b/Project.toml index b1ef7807..62d82f58 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "GPUCompiler" uuid = "61eb1bfa-7361-4325-ad38-22787b887f55" -version = "2.1.1" +version = "2.2.0" authors = ["Tim Besard "] [workspace] diff --git a/src/GPUCompiler.jl b/src/GPUCompiler.jl index d1ef48e2..98bb256d 100644 --- a/src/GPUCompiler.jl +++ b/src/GPUCompiler.jl @@ -54,6 +54,7 @@ include("mangling.jl") # compiler interface and implementations include("interface.jl") +include("relocation.jl") include("error.jl") include("native.jl") include("ptx.jl") @@ -84,13 +85,10 @@ include("precompile.jl") function __init__() STDERR_HAS_COLOR[] = get(stderr, :color, false) + empty!(session_results_cache) @static if !HAS_INTEGRATED_CACHE - # session-local results keyed by CodeInstance; entries serialized during - # GPUCompiler's own precompilation can never be valid in a later session - empty!(legacy_job_results) - # ditto for the in-process CodeCaches: CIs deposited by our own precompile - # workload carry world ages from the precompilation process + # CodeInstances created by GPUCompiler's precompile workload are process-local. empty!(GLOBAL_CI_CACHES) end diff --git a/src/deprecated.jl b/src/deprecated.jl index d3c649eb..e1b80f77 100644 --- a/src/deprecated.jl +++ b/src/deprecated.jl @@ -229,35 +229,6 @@ function CC.findsup(@nospecialize(sig::Type), table::StackedMethodTable) end -## 1.10 `cached_results` -# -# Session-local storage for the per-job results structs; on 1.11+ these live on the -# `CodeInstance`s of Julia's integrated cache instead (see `interface.jl`). Keep the -# same identity here by associating results with a foreign `CodeInstance`: unrelated -# world-age advances can reuse a still-valid CI, while invalidation makes the lookup -# resolve to a new CI and therefore a new results struct. -struct LegacyJobResultEntry - config::CompilerConfig - value::Any -end - -const legacy_job_results = IdDict{CodeInstance,Vector{LegacyJobResultEntry}}() -const job_results_lock = ReentrantLock() - -function job_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} - Base.@lock job_results_lock begin - entries = get!(legacy_job_results, ci) do - LegacyJobResultEntry[] - end - for entry in entries - entry.config === config && entry.value isa V && return entry.value::V - end - v = V() - push!(entries, LegacyJobResultEntry(config, v)) - return v - end -end - function job_code_instance(@nospecialize(job::CompilerJob)) cache = WorldView(get_code_cache(job), job.world, job.world) CC.get(cache, job.source, nothing) @@ -268,18 +239,9 @@ end function cached_results(::Type{V}, job::CompilerJob) where {V} ci = job_code_instance(job) ci === nothing && return nothing - return job_results(V, ci, job.config) + return session_results(V, ci, job.config) end - -## 1.10 session-dependent results -# -# Nothing to wipe: `legacy_job_results` never persists meaningfully across sessions (its -# CodeInstance keys are session-specific, and our own image's entries are cleared in -# `__init__`; entries written by a downstream package's workload don't make it into that -# package's image at all, cf. cross-image mutation loss). -mark_session_dependent!(@nospecialize(job::CompilerJob)) = nothing - end # !HAS_INTEGRATED_CACHE diff --git a/src/driver.jl b/src/driver.jl index 38581053..4389abac 100644 --- a/src/driver.jl +++ b/src/driver.jl @@ -50,6 +50,10 @@ export compile Compile a `job` to one of the following formats as specified by the `target` argument: `:llvm` for LLVM IR, `:asm` for assembly, or `:obj` for object code. + +The default [`relocation_lowering`](@ref) strategy resolves Julia-value relocations in the +`:llvm` result. Other strategies retain relocation metadata for their loader; `:defer` +consumers resolve it with [`apply_relocations!`](@ref). """ function compile(target::Symbol, @nospecialize(job::CompilerJob)) if compile_hook[] !== nothing @@ -58,7 +62,8 @@ function compile(target::Symbol, @nospecialize(job::CompilerJob)) return compile_unhooked(target, job) end -function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) +function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob); + resolve_relocations::Bool=true) if context(; throw_error=false) === nothing error("No active LLVM context. Use `JuliaContext()` do-block syntax to create one.") end @@ -73,7 +78,7 @@ function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) ## LLVM IR - ir, ir_meta = emit_llvm(job) + ir, ir_meta = emit_llvm(job; resolve_relocations) if output == :llvm if job.config.strip @@ -93,7 +98,7 @@ function compile_unhooked(output::Symbol, @nospecialize(job::CompilerJob)) else error("Unknown assembly format $output") end - asm, asm_meta = emit_asm(job, ir, format) + asm, asm_meta = emit_asm(job, ir, ir_meta.relocations, format) if output == :asm || output == :obj return asm, (; asm_meta..., ir_meta..., ir) @@ -169,7 +174,8 @@ end const __llvm_initialized = Ref(false) -@locked function emit_llvm(@nospecialize(job::CompilerJob)) +@locked function emit_llvm(@nospecialize(job::CompilerJob); + resolve_relocations::Bool=true) if !__llvm_initialized[] InitializeAllTargets() InitializeAllTargetInfos() @@ -180,7 +186,7 @@ const __llvm_initialized = Ref(false) end @tracepoint "IR generation" begin - ir, compiled, gv_to_value = irgen(job) + ir, compiled, relocations = irgen(job) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -245,11 +251,9 @@ const __llvm_initialized = Ref(false) dyn_ir, dyn_meta = @invokelatest deferred_codegen(dyn_job, job) dyn_entry_fn = LLVM.name(dyn_meta.entry) merge!(compiled, dyn_meta.compiled) - if haskey(dyn_meta, :gv_to_value) - merge!(gv_to_value, dyn_meta.gv_to_value) - end @assert context(dyn_ir) == context(ir) - link!(ir, dyn_ir) + link_relocatable!(ir, relocations, dyn_ir, + dyn_meta.relocations) changed = true dyn_entry_fn end @@ -292,7 +296,7 @@ const __llvm_initialized = Ref(false) if job.config.toplevel && job.config.libraries # load the runtime outside of a timing block (because it recurses into the compiler) if !uses_julia_runtime(job) - runtime = load_runtime(job) + runtime, runtime_relocs = load_runtime(job) end @tracepoint "Library linking" begin @@ -301,7 +305,8 @@ const __llvm_initialized = Ref(false) # GPU run-time library if !uses_julia_runtime(job) - @tracepoint "runtime library" link!(ir, runtime; only_needed=true) + @tracepoint "runtime library" link_relocatable!( + ir, relocations, runtime, runtime_relocs; only_needed=true) end end end @@ -334,13 +339,16 @@ const __llvm_initialized = Ref(false) finish_linked_module!(job, ir) - # Materialize isbits and Bool boxes; bake addresses for other objects. - portable = relocate_gvs!(ir, gv_to_value) - portable || mark_session_dependent!(job) + # Resolve early so optimization sees concrete values. + resolve_early = resolve_relocations && relocation_lowering(job) === :bake + if resolve_early + prune_dead_relocations!(ir, relocations) + bake_relocations!(ir, relocations) + end if job.config.optimize @tracepoint "optimization" begin - optimize!(job, ir; job.config.opt_level) + optimize!(job, ir, relocations; job.config.opt_level) # deferred codegen has some special optimization requirements, # which also need to happen _after_ regular optimization. @@ -362,6 +370,12 @@ const __llvm_initialized = Ref(false) end end + # Runtime linking during optimization can add relocations. + if resolve_early && !isempty(relocations.sites) + prune_dead_relocations!(ir, relocations) + bake_relocations!(ir, relocations) + end + if job.config.cleanup @tracepoint "clean-up" begin @dispose pb=NewPMPassBuilder() begin @@ -375,6 +389,9 @@ const __llvm_initialized = Ref(false) end end + # Do not expose dead sites to loaders. + resolve_early || prune_dead_relocations!(ir, relocations) + # optimization may have replaced functions, so look the entry point up again entry = functions(ir)[entry_fn] @@ -405,7 +422,7 @@ const __llvm_initialized = Ref(false) if job.config.toplevel && job.config.validate @tracepoint "validation" begin - check_ir(job, ir) + check_ir(job, ir, relocations) end end @@ -413,18 +430,23 @@ const __llvm_initialized = Ref(false) @tracepoint "verification" verify(ir) end - return ir, (; entry, compiled, gv_to_value) + return ir, (; entry, compiled, relocations) end +# Compatibility for back-ends that resolve relocations during `emit_llvm`. +emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, + format::LLVM.API.LLVMCodeGenFileType) = + emit_asm(job, ir, Relocations(), format) + @locked function emit_asm(@nospecialize(job::CompilerJob), ir::LLVM.Module, - format::LLVM.API.LLVMCodeGenFileType) + relocs::Relocations, format::LLVM.API.LLVMCodeGenFileType) # NOTE: strip after validation to get better errors if job.config.strip @tracepoint "Debug info removal" strip_debuginfo!(ir) end @tracepoint "LLVM back-end" begin - @tracepoint "preparation" prepare_execution!(job, ir) + @tracepoint "preparation" prepare_execution!(job, ir, relocs) code = @tracepoint "machine-code generation" mcgen(job, ir, format) end diff --git a/src/interface.jl b/src/interface.jl index f17c6e3c..22ba6622 100644 --- a/src/interface.jl +++ b/src/interface.jl @@ -302,6 +302,23 @@ can_vectorize(@nospecialize(job::CompilerJob)) = false # Should emit PTLS lookup that can be relocated dump_native(@nospecialize(job::CompilerJob)) = false +""" + relocation_lowering(job) -> Symbol + +Select how a back-end lowers relocations: + +- `:bake` resolves them in `emit_llvm`. +- `:patch` emits definitions for the loader to patch. +- `:import` emits declarations for the loader to resolve; interior sites still need patching. +- `:defer` leaves the `:llvm` result for [`apply_relocations!`](@ref). Object emission + requires all relocations to have been applied. + +Relocatable output requires [`supports_relocatable_ir`](@ref). Resolving relocations +permanently roots the referenced Julia values in the process (mirroring Julia's own +codegen), so loaders need no GC bookkeeping of their own. +""" +relocation_lowering(@nospecialize(job::CompilerJob)) = :bake + # the Julia module to look up target-specific runtime functions in (this includes both # target-specific functions from the GPU runtime library, like `malloc`, but also # replacements functions for operations like `Base.sin`) @@ -376,6 +393,33 @@ else cache_owner(job.config.target, job.config.params, job.config.always_inline) end +struct SessionResultEntry + config::CompilerConfig + value::Any +end + +const session_results_cache = IdDict{CodeInstance,Vector{SessionResultEntry}}() +const session_results_lock = ReentrantLock() + +function session_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} + Base.@lock session_results_lock begin + entries = get!(session_results_cache, ci) do + SessionResultEntry[] + end + for entry in entries + entry.config === config && entry.value isa V && return entry.value::V + end + value = V() + push!(entries, SessionResultEntry(config, value)) + return value + end +end + +function can_persist_results(@nospecialize(job::CompilerJob)) + supports_relocatable_ir() && + relocation_lowering(job) in (:patch, :import, :defer) +end + """ cached_results(::Type{V}, job::CompilerJob) -> Union{Nothing,V} @@ -407,21 +451,13 @@ post-compile lookup in the example is guaranteed to succeed. To attach results w generating code — e.g. from an inference-only precompilation workload — run `precompile(job)` first. -Results are keyed by the *full* compiler job: its method instance, world age, and entire -`CompilerConfig` (two jobs differing only in, say, the kernel `name` get distinct -structs). Storage differs per Julia version, transparently to the back-end: +Results are keyed by the method instance, world age, and full `CompilerConfig`. On Julia +1.11+, results persist with the `CodeInstance` only when the back-end selects `:patch`, +`:import`, or `:defer` and [`supports_relocatable_ir`](@ref) is true. Other results use a +session-local store. Julia 1.10 always uses the session-local store. -- On Julia 1.11+, the struct lives on the `CodeInstance`'s `analysis_results` chain in - Julia's integrated code cache, partitioned by [`cache_owner`](@ref) and keyed by - config within the per-CI [`JobResults`](@ref) container. Method redefinition - invalidates the CI — and with it the attached results. Artifacts stored during - precompilation are serialized into the package image along with the CI: populate only - session-portable values (bytes, strings) during precompile workloads, and keep - session-local handles (device modules, pipeline objects) in fields that remain empty - until first use at run time. - -- On Julia 1.10, the struct lives in a session-local store keyed by the foreign - `CodeInstance` and config. Nothing persists across sessions. +Persistent result structs may contain session-local handles, but back-ends must leave those +fields empty during precompilation. Thread safety: concurrent calls for the same job return the same struct, but GPUCompiler does not serialize back-end *compilation*; take a back-end lock around @@ -432,7 +468,7 @@ function cached_results end @static if HAS_INTEGRATED_CACHE """ - JobResults + PersistentJobResults Per-CodeInstance container mapping a `CompilerConfig` to the back-end's results struct. @@ -447,28 +483,28 @@ from package images. # `CompilerConfig` is an abstract UnionAll here. Keeping it in a tuple stored inline in a # Vector boxes the config again on every iteration. A non-isbits entry object is stored by # reference, so both fields are boxed once and hot-path scans allocate nothing. -struct JobResultEntry +struct PersistentResultEntry config::CompilerConfig value::Any end -mutable struct JobResults - entries::Vector{JobResultEntry} - JobResults() = new(JobResultEntry[]) +mutable struct PersistentJobResults + entries::Vector{PersistentResultEntry} + PersistentJobResults() = new(PersistentResultEntry[]) end -const cached_results_lock = ReentrantLock() +const persistent_results_lock = ReentrantLock() # NOTE: like `cache_owner`, specialized for the launch hot path (bounded number of # instantiations: one per back-end and results type). -function job_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} - jr = CompilerCaching.results(JobResults, ci) - Base.@lock cached_results_lock begin - for entry in jr.entries +function persistent_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} + results = CompilerCaching.results(PersistentJobResults, ci) + Base.@lock persistent_results_lock begin + for entry in results.entries entry.config === config && entry.value isa V && return entry.value::V end v = V() - push!(jr.entries, JobResultEntry(config, v)) + push!(results.entries, PersistentResultEntry(config, v)) return v end end @@ -476,7 +512,7 @@ end function cache_view(@nospecialize(job::CompilerJob)) # `cache_owner` is deliberately stored as Any on CompilerConfig: preserve that box in the # CacheView instead of re-specializing and re-boxing the immutable token for the ccall. - CompilerCaching.CacheView{Any,JobResults}(cache_owner(job), job.world) + CompilerCaching.CacheView{Any,PersistentJobResults}(cache_owner(job), job.world) end # Fetch the CodeInstance backing `job` from the integrated cache. On 1.14+, inference @@ -501,54 +537,18 @@ end function cached_results(::Type{V}, job::CompilerJob) where {V} ci = job_code_instance(job) ci === nothing && return nothing - return job_results(V, ci, job.config) -end - -## session-dependent results -# -# Some compilation results embed session-specific data: `relocate_gvs!` bakes absolute -# pointers into the IR of toplevel jobs that reference `julia.constgv` globals (except -# for slots it can materialize as session-portable device constants), and any -# artifact a back-end derives from that IR (metallib, SPIR-V, ...) inherits them. Such -# results must not survive into a package image, while remaining available for -# within-session lookups during the precompilation process itself. Julia wipes its own -# session-dependent CodeInstance state during serialization (staticdata.c); we -# approximate that with an `atexit` hook, which the runtime invokes *before* -# `jl_write_compiler_output`: right before the image is written, the entries of jobs -# marked session-dependent are deleted from their `JobResults` container, so a later -# session simply recompiles them. - -const session_dependent_jobs = Vector{CompilerJob}() -const session_dependent_lock = ReentrantLock() - -function mark_session_dependent!(@nospecialize(job::CompilerJob)) - ccall(:jl_generating_output, Cint, ()) == 1 || return - Base.@lock session_dependent_lock begin - if isempty(session_dependent_jobs) - atexit(wipe_session_dependent_results) - end - push!(session_dependent_jobs, job) - end - return -end - -function wipe_session_dependent_results() - Base.@lock session_dependent_lock begin - for job in session_dependent_jobs - ci = job_code_instance(job) - ci === nothing && continue - jr = CompilerCaching.results(JobResults, ci) - Base.@lock cached_results_lock begin - filter!(entry -> entry.config !== job.config, jr.entries) - end - end - empty!(session_dependent_jobs) + if can_persist_results(job) + return persistent_results(V, ci, job.config) + else + return session_results(V, ci, job.config) end - return end end # HAS_INTEGRATED_CACHE +# Public relocation interface. +@public RelocationSite, Relocations, relocation_lowering +@public apply_relocations!, resolved_relocations, supports_relocatable_ir @public GPUCompilerCacheToken, cache_owner, cached_results # the method table to use diff --git a/src/irgen.jl b/src/irgen.jl index 9129e20c..648459be 100644 --- a/src/irgen.jl +++ b/src/irgen.jl @@ -2,6 +2,7 @@ function irgen(@nospecialize(job::CompilerJob)) mod, compiled, gv_to_value = @tracepoint "emission" compile_method_instance(job) + relocations = collect_julia_value_relocations!(mod, gv_to_value) if job.config.entry_abi === :specfunc entry_fn = compiled[job.source].specfunc else @@ -43,18 +44,22 @@ function irgen(@nospecialize(job::CompilerJob)) end end - # sanitize global values (Julia doesn't when using the external codegen policy) - for val in [collect(globals(mod)); collect(functions(mod))] + # sanitize defined globals (external declaration names are part of their ABI) + for val in collect(globals(mod)) + isdeclaration(val) && continue + new_name = safe_name(LLVM.name(val)) + if LLVM.name(val) != new_name + LLVM.name!(val, new_name) + end + end + + # sanitize defined functions (Julia doesn't when using the external codegen policy) + for val in collect(functions(mod)) isdeclaration(val) && continue old_name = LLVM.name(val) new_name = safe_name(old_name) if old_name != new_name LLVM.name!(val, new_name) - val = get(gv_to_value, old_name, nothing) - if val !== nothing - delete!(gv_to_value, old_name) - gv_to_value[new_name] = val - end end end @@ -145,7 +150,7 @@ function irgen(@nospecialize(job::CompilerJob)) lower_alloca!(job, mod) end - return mod, compiled, gv_to_value + return mod, compiled, relocations end diff --git a/src/jlgen.jl b/src/jlgen.jl index d4820377..93691d55 100644 --- a/src/jlgen.jl +++ b/src/jlgen.jl @@ -563,14 +563,10 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) end end - # Maintain a map from global variables to their initialized Julia values. The - # objects pointed to are perma-rooted during codegen. We *don't* bake these - # addresses into the IR yet, so that we can cache it across sessions. + # Map global variables to their rooted Julia values without embedding addresses in IR. gv_to_value = Dict{String, Ptr{Cvoid}}() if gvs === nothing - # No reliable GV table on this Julia — best-effort discovery from the module. - # On these older versions Julia's own emit may have already baked in absolute - # pointer values; we recover them by reading existing initializers. + # Without a reliable GV table, recover addresses from existing initializers. for gv in globals(llvm_mod) if !haskey(metadata(gv), "julia.constgv") continue @@ -597,15 +593,8 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) gv = GlobalVariable(gv_ref) gv_to_value[LLVM.name(gv)] = init end - # Strip the initializers so the IR we hand back is session-portable - # (on 1.12 `jl_emit_native_impl` bakes pointers via - # `literal_static_pointer_val`; on 1.13+ Julia nulls them itself); - # `relocate_gvs!` at the toplevel link step writes the session-current - # value in. Demote each GV to an external *declaration* rather than - # giving it a null initializer: an internal global initialized to null - # is fair game for optimization passes that run before relocation - # (e.g. GlobalOpt in the PTX back-end's `finish_module!`), which would - # fold loads of it to null and delete the global. + # Discard session addresses. Declarations survive optimization until relocation + # lowering; null definitions could be folded away first. for gv in globals(llvm_mod) haskey(gv_to_value, LLVM.name(gv)) || continue initializer!(gv, nothing) @@ -756,131 +745,6 @@ function compile_method_instance(@nospecialize(job::CompilerJob)) return llvm_mod, compiled, gv_to_value end -""" - relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - -Resolve globals that refer to Julia objects. `jl_true`/`jl_false`, which are -absent from `gv_to_value`, become canonical module-local boxes. Ordinary -non-ghost isbits objects are also materialized; other objects keep their host -address as an opaque identity token. - -Only GVs that are declarations (as produced by `compile_method_instance`, -which strips the initializers for session portability) or still have a null -initializer are touched: preserving existing initializers covers the -older-Julia path where Julia itself emits pointer values directly. - -Apart from the dedicated Bool globals, GVs present in `mod` but missing from -`gv_to_value` remain declarations, which back-ends will reject loudly -(undefined symbol) rather than silently folding to null. - -Returns `true` if the module is session-portable afterwards: no absolute host -address was written (neither a baked slot nor a materialized header carrying -a non-smalltag type pointer). -""" -function relocate_gvs!(mod::LLVM.Module, gv_to_value::Dict{String, Ptr{Cvoid}}) - portable = materialize_bool_singletons!(mod) - mod_gvs = globals(mod) - for (name, init) in gv_to_value - # Bools are resolved by name above. - name in ("jl_true", "jl_false") && continue - - haskey(mod_gvs, name) || continue - gv = mod_gvs[name] - cur = initializer(gv) - if !(cur === nothing || LLVM.isnull(cur)) - # pre-baked by Julia itself (pre-1.13): also session-absolute - portable = false - continue - end - - val = nothing - if init != C_NULL - obj = Base.unsafe_pointer_to_objref(init) - # Zero-sized objects remain identity tokens. - if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) - val, hdr = materialize_box!(mod, gv, obj, init) - # non-smalltag headers carry a host DataType pointer - portable &= hdr < UInt(64 << 4) # jl_max_tags << 4 - end - end - if val === nothing - val = const_inttoptr(ConstantInt(Int64(init)), global_value_type(gv)) - portable = false - end - initializer!(gv, val) - # re-internalize what compile_method_instance demoted to an external - # declaration; with the value in place, the optimizer can now fold it - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - end - return portable -end - -# Bool JuliaVariables are absent from `gv_to_value`; define one device box per name. -function materialize_bool_singletons!(mod::LLVM.Module) - portable = true - mod_gvs = globals(mod) - for (name, obj) in ("jl_true" => true, "jl_false" => false) - haskey(mod_gvs, name) || continue - gv = mod_gvs[name] - cur = initializer(gv) - if !(cur === nothing || LLVM.isnull(cur)) - # Existing definitions may contain session-specific addresses. - portable = false - continue - end - - init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) - val, hdr = materialize_box!(mod, gv, obj, init) - initializer!(gv, val) - constant!(gv, true) - linkage!(gv, LLVM.API.LLVMPrivateLinkage) - - # Stay conservative if Bool stops using a smalltag. - portable &= hdr < UInt(64 << 4) # jl_max_tags << 4 - end - return portable -end - -# emit a device-resident constant replica of the box holding `obj`; returns -# the constant to store in the slot, and the (gcbits-masked) header word -function materialize_box!(mod::LLVM.Module, gv::GlobalVariable, @nospecialize(obj), - init::Ptr{Cvoid}) - @assert isbitstype(typeof(obj)) && sizeof(obj) > 0 - - W = sizeof(Int) - hdr, bytes = GC.@preserve obj begin - # the header word transparently yields the smalltag immediate for - # smalltag types and the host type pointer otherwise; drop the gcbits - hdr = unsafe_load(Ptr{UInt}(init - W)) & ~UInt(15) - bytes = [unsafe_load(Ptr{UInt8}(init), i) for i in 1:sizeof(obj)] - hdr, bytes - end - - T_word = LLVM.IntType(8W) - T_byte = LLVM.Int8Type() - fields = LLVM.Constant[ConstantInt(T_word, hdr), ConstantDataArray(T_byte, bytes)] - payload_idx = 1 - if Base.datatype_alignment(typeof(obj)) > W - # pad so the payload lands at a 16-byte offset (JL_HEAP_ALIGNMENT max) - pushfirst!(fields, ConstantDataArray(T_byte, zeros(UInt8, 16 - W))) - payload_idx = 2 - end - boxinit = ConstantStruct(fields) - boxty = value_type(boxinit) - - box = GlobalVariable(mod, boxty, safe_name(LLVM.name(gv)) * "_box") - initializer!(box, boxinit) - constant!(box, true) - linkage!(box, LLVM.API.LLVMPrivateLinkage) - alignment!(box, 16) - unnamed_addr!(box, true) - - idx(i) = ConstantInt(LLVM.Int32Type(), i) - payload = const_gep(boxty, box, LLVM.Constant[idx(0), idx(payload_idx)]) - slotty = global_value_type(gv) - val = value_type(payload) == slotty ? payload : const_addrspacecast(payload, slotty) - return val, hdr -end # partially revert JuliaLang/julia#49391 — see #527 @static if v"1.11.0-DEV.1603" <= VERSION < v"1.12.0-DEV.347" && # reverted on master diff --git a/src/mcgen.jl b/src/mcgen.jl index 3fb4a5c6..0dda9ecc 100644 --- a/src/mcgen.jl +++ b/src/mcgen.jl @@ -1,79 +1,33 @@ # machine code generation -# final preparations for the module to be compiled to machine code -# these passes should not be run when e.g. compiling to write to disk. -function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module) - @dispose pb=NewPMPassBuilder() begin - register!(pb, ResolveCPUReferencesPass(job)) - - add!(pb, RecomputeGlobalsAAPass()) - add!(pb, GlobalOptPass()) - add!(pb, ResolveCPUReferencesPass(job)) - add!(pb, GlobalDCEPass()) - add!(pb, StripDeadPrototypesPass()) - - run!(pb, mod, llvm_machine(job.config.target)) +# Finalize the module for backend emission by collecting and lowering all live relocations. +function prepare_execution!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + relocs::Relocations=Relocations()) + # Clean up first so only live relocations get lowered. + function cleanup() + @dispose pb=NewPMPassBuilder() begin + add!(pb, RecomputeGlobalsAAPass()) + add!(pb, GlobalOptPass()) + add!(pb, GlobalDCEPass()) + add!(pb, StripDeadPrototypesPass()) + run!(pb, mod, llvm_machine(job.config.target)) + end end + cleanup() + prune_dead_relocations!(mod, relocs) - return -end - -# some Julia code contains references to objects in the CPU run-time, -# without actually using the contents or functionality of those objects. -# -# prime example are type tags, which reference the address of the allocated type. -# since those references are ephemeral, we can't eagerly resolve and emit them in the IR, -# but at the same time the GPU can't resolve them at run-time. -# -# this pass performs that resolution at link time. -struct ResolveCPUReferences - job::CompilerJob -end -function (self::ResolveCPUReferences)(mod::LLVM.Module) - changed = false + collect_cglobal_relocations!(mod, relocs) + lower_relocations!(job, mod, relocs) - for f in functions(mod) - fn = LLVM.name(f) - if isdeclaration(f) && !LLVM.isintrinsic(f) && startswith(fn, "jl_") - # lazily resolve the address of the binding; some symbols only exist - # within the JIT (e.g. `jl_get_pgcstack_resolved`) and cannot be looked up, - # but such symbols are only ever called, not loaded from. - dereferenced = nothing - function resolve_binding() - if dereferenced === nothing - address = ccall(:jl_cglobal, Any, (Any, Any), fn, UInt) - dereferenced = LLVM.ConstantInt(unsafe_load(address)) - end - dereferenced - end + # Fold constants exposed by eager lowering, and discard slots made dead by + # either lowering strategy. + cleanup() + prune_dead_relocations!(mod, relocs) - function replace_bindings!(value) - changed = false - for use in uses(value) - val = user(use) - if isa(val, LLVM.ConstantExpr) - # recurse - changed |= replace_bindings!(val) - elseif isa(val, LLVM.LoadInst) - # resolve - replace_uses!(val, resolve_binding()) - erase!(val) - # FIXME: iterator invalidation? - changed = true - end - end - changed - end - - changed |= replace_bindings!(f) - end - end - - return changed + has_unresolved_cglobal_loads(mod, relocs) && + error("Unresolved cglobal load after relocation lowering") + return end -ResolveCPUReferencesPass(job) = - NewPMModulePass("ResolveCPUReferences", ResolveCPUReferences(job)) - function mcgen(@nospecialize(job::CompilerJob), mod::LLVM.Module, format=LLVM.API.LLVMAssemblyFile) tm = llvm_machine(job.config.target) diff --git a/src/optim.jl b/src/optim.jl index b0b339a3..932f316b 100644 --- a/src/optim.jl +++ b/src/optim.jl @@ -49,7 +49,8 @@ function aggressiveinstcombine_pass(@nospecialize(job::CompilerJob)) end end -function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module; opt_level=2) +function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + relocs::Relocations; opt_level=2) tm = llvm_machine(job.config.target) tti = llvm_targetinfo(job.config.target) @@ -59,7 +60,7 @@ function optimize!(@nospecialize(job::CompilerJob), mod::LLVM.Module; opt_level= register!(pb, GPULowerCPUFeaturesPass(job)) register!(pb, GPULowerPTLSPass(job)) register!(pb, GPULowerGCFramePass(job)) - register!(pb, GPULinkRuntimePass(job)) + register!(pb, GPULinkRuntimePass(job, relocs)) register!(pb, GPULinkLibrariesPass(job)) register!(pb, GPUFinishRuntimeIntrinsicsPass(job)) register!(pb, AddKernelStatePass(job)) @@ -325,7 +326,8 @@ function buildIntrinsicLoweringPipeline(mpm, @nospecialize(job::CompilerJob), op add!(fpm, GPULowerGCFramePass(job)) end if job.config.libraries - add!(mpm, GPULinkRuntimePass(job)) + # Use the registered pass because it owns `relocs`; the others only capture `job`. + add!(mpm, "GPULinkRuntime") add!(mpm, GPULinkLibrariesPass(job)) add!(mpm, GPUFinishRuntimeIntrinsicsPass(job)) end @@ -475,6 +477,7 @@ GPULowerCPUFeaturesPass(job) = NewPMModulePass("GPULowerCPUFeatures", CPUFeature struct LinkRuntime job::CompilerJob + relocs::Relocations end function (self::LinkRuntime)(mod::LLVM.Module) self.job.config.libraries || return false @@ -483,15 +486,16 @@ function (self::LinkRuntime)(mod::LLVM.Module) # GC lowering can introduce new calls to GPU runtime functions after the runtime # was linked initially. Link again now so those calls resolve to definitions before # later intrinsic-lowering passes inspect or rewrite the runtime call graph. - runtime = load_runtime(self.job) + runtime, runtime_relocs = load_runtime(self.job) # `RemoveNIPass` stripped non-integral address spaces from `mod`'s datalayout, but the # cached runtime kept them; align it (as with target libraries) to avoid a warning. triple!(runtime, triple(mod)) datalayout!(runtime, datalayout(mod)) - link!(mod, runtime; only_needed=true) + link_relocatable!(mod, self.relocs, runtime, runtime_relocs; only_needed=true) return true end -GPULinkRuntimePass(job) = NewPMModulePass("GPULinkRuntime", LinkRuntime(job)) +GPULinkRuntimePass(job, relocs::Relocations) = + NewPMModulePass("GPULinkRuntime", LinkRuntime(job, relocs)) struct LinkLibraries job::CompilerJob diff --git a/src/relocation.jl b/src/relocation.jl new file mode 100644 index 00000000..30fc65ab --- /dev/null +++ b/src/relocation.jl @@ -0,0 +1,591 @@ +# Relocations map words in globals to Julia values or named C globals. Declarations can be +# resolved, patched, or imported; sites inside definitions can only be resolved or patched. +# +# produce ──▶ merge (on link) ──▶ prune (after DCE) ──▶ lower +# +# `:patch` and `:import` need a symbol namespace per object. `:defer` instead applies sites +# directly to each parsed module. Site names are fixed at creation so IR and metadata can be +# linked without renaming. + + +## targets + +""" + JuliaValueRef(value) + +A Julia value with a stable address (a heap object, symbol, or singleton), used as the +serializable identity of a relocation target. Resolve it in the active session with +[`resolve_relocation_target`](@ref), which permanently roots the value in the process so +the resolved address stays valid for as long as the session lives. +""" +struct JuliaValueRef + value::Any + + function JuliaValueRef(value) + isbitstype(typeof(value)) && sizeof(value) > 0 && + error("JuliaValueRef requires an object with a stable address") + new(value) + end +end + +""" + CGlobalRef(symbol, library=nothing) + +A named C data global. With `library === nothing`, resolution uses `jl_cglobal`'s +process-wide lookup. Otherwise it looks up `symbol` in `library`. Resolution returns the +word stored in the global. +""" +struct CGlobalRef + symbol::Symbol + library::Union{Nothing,String} +end +CGlobalRef(symbol::Symbol) = CGlobalRef(symbol, nothing) + +""" + RelocationTarget + +A serializable target for a relocated word: either a [`JuliaValueRef`](@ref) or a +[`CGlobalRef`](@ref). +""" +const RelocationTarget = Union{JuliaValueRef,CGlobalRef} + +same_relocation_target(a::JuliaValueRef, b::JuliaValueRef) = a.value === b.value +same_relocation_target(a::CGlobalRef, b::CGlobalRef) = + a.symbol === b.symbol && a.library == b.library +same_relocation_target(::RelocationTarget, ::RelocationTarget) = false + +# Permanently root a value in the current process and return the canonical rooted +# instance, exactly as Julia's own codegen does for values referenced from native code +# (`jl_ensure_rooted` on 1.10/1.11, `aot_optimize_roots` on 1.12+). Values compiled in +# this session are already rooted this way, making this a cheap lookup; it matters for +# metadata deserialized from a cache, whose values codegen never saw. Rooting by egal +# identity also folds such duplicates onto the instance native code already uses. +function root_relocation_target(target::JuliaValueRef) + @static if VERSION >= v"1.11-" + ccall(:jl_as_global_root, Any, (Any, Cint), target.value, 1) + else + ccall(:jl_as_global_root, Any, (Any,), target.value) + end +end + +value_pointer(@nospecialize(value)) = UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), value)) + +""" + resolve_relocation_target(target) -> UInt + +Resolve a relocation target to its word in the current Julia process. Julia values are +permanently rooted (and canonicalized by egal identity) as part of resolution, so the +returned address cannot dangle. +""" +function resolve_relocation_target(target::JuliaValueRef) + value_pointer(root_relocation_target(target)) +end +function resolve_relocation_target(target::CGlobalRef) + if target.library === nothing + # `jl_cglobal` accepts the symbol directly and does the process-wide `jl_dlfind`. + address = ccall(:jl_cglobal, Any, (Any, Any), target.symbol, UInt) + return unsafe_load(address) + end + handle = Libdl.dlopen(target.library) + address = Libdl.dlsym(handle, target.symbol) + return unsafe_load(Ptr{UInt}(address)) +end + + +## the table + +""" + RelocationSite(name, offset) + +The location of a word to relocate: a global name and a byte offset within that global. +The global's IR shape determines how the site is lowered. A declaration denotes an +importable word-sized slot and must have offset zero; a definition denotes a post-load +patch within its initializer. +""" +struct RelocationSite + name::String + offset::Int + + function RelocationSite(name::String, offset::Int) + offset >= 0 || throw(ArgumentError("relocation offset must be nonnegative")) + new(name, offset) + end +end + +""" + Relocations(sites) + +Relocation metadata accompanying a module. `sites` maps [`RelocationSite`](@ref)s to +targets. See [`resolved_relocations`](@ref) for resolving sites for a loader. +""" +struct Relocations + sites::Dict{RelocationSite,RelocationTarget} +end + +Relocations() = Relocations(Dict{RelocationSite,RelocationTarget}()) + +# Resolving into IR consumes the site table; loaders copy cached metadata first. +Base.copy(relocs::Relocations) = Relocations(copy(relocs.sites)) + +""" + resolved_relocations(relocs) -> Vector{Pair{RelocationSite,UInt}} + +Resolve relocation metadata for a loader, returning each site with its resolved word. +Resolution permanently roots referenced Julia values in the process, so the addresses +stay valid for the lifetime of the session. +""" +function resolved_relocations(relocs::Relocations) + sites = Pair{RelocationSite,UInt}[] + for (site, ref) in relocs.sites + push!(sites, site => resolve_relocation_target(ref)) + end + return sites +end + +relocation_word_type() = LLVM.IntType(8sizeof(UInt)) + +function check_slot_size(mod::LLVM.Module, gv::GlobalVariable, name::String) + size = abi_size(datalayout(mod), global_value_type(gv)) + size == sizeof(UInt) || + error("Relocation slot '$name' has size $size, expected $(sizeof(UInt))") + return +end + +function slot_initializer(gv::GlobalVariable, value::UInt) + T = global_value_type(gv) + if T isa LLVM.PointerType + return const_inttoptr(ConstantInt(UInt64(value)), T) + elseif T isa LLVM.IntegerType && width(T) == 8sizeof(UInt) + return ConstantInt(T, value) + end + error("Relocation slot '$(LLVM.name(gv))' has unsupported LLVM type $T") +end + +function foreach_relocation(f, mod::LLVM.Module, relocs::Relocations) + mod_gvs = globals(mod) + for (site, ref) in relocs.sites + name = site.name + offset = site.offset + haskey(mod_gvs, name) || error("Missing relocation global '$name'") + gv = mod_gvs[name] + if isdeclaration(gv) + offset == 0 || + error("Relocation declaration '$name' has nonzero offset $offset") + check_slot_size(mod, gv, name) + else + init = initializer(gv) + init === nothing && error("Relocation global '$name' has no initializer") + T = value_type(init) + T isa LLVM.StructType || + error("Relocation global '$name' has non-struct initializer $T") + size = abi_size(datalayout(mod), T) + offset + sizeof(UInt) <= size || + error("Relocation '$name+$offset' is outside its $size-byte global") + end + f(site, gv, ref) + end + return +end + + +## producers + +function collect_julia_value_relocations!(mod::LLVM.Module, + gv_to_value::Dict{String, Ptr{Cvoid}}) + relocs = Relocations() + mod_gvs = globals(mod) + for (name, init) in gv_to_value + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + cur = initializer(gv) + if !(cur === nothing || LLVM.isnull(cur)) + @assert !supports_relocatable_ir() + continue + end + + # jl_get_llvm_gvs and jl_get_llvm_gv_inits report an initializer for every + # mapped global, so a null here means those maps are out of sync. + init == C_NULL && error("Missing Julia object for global '$name'") + obj = Base.unsafe_pointer_to_objref(init) + if isbitstype(typeof(obj)) && sizeof(obj) > 0 && !(obj isa Bool) + val = materialize_box!(mod, relocs, gv, obj, init) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + else + check_slot_size(mod, gv, name) + slot_name = safe_name(name) * "_" * string(objectid(obj); base=16) + LLVM.name!(gv, slot_name) + LLVM.name(gv) == slot_name || + error("Relocation slot name '$slot_name' is already in use") + relocs.sites[RelocationSite(slot_name, 0)] = JuliaValueRef(obj) + end + end + + # Bool JuliaVariables are absent from `gv_to_value`; define one device box per module. + for (name, obj) in ("jl_true" => true, "jl_false" => false) + haskey(mod_gvs, name) || continue + gv = mod_gvs[name] + cur = initializer(gv) + if !(cur === nothing || LLVM.isnull(cur)) + @assert !supports_relocatable_ir() + continue + end + + init = ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), obj) + val = materialize_box!(mod, relocs, gv, obj, init) + initializer!(gv, val) + constant!(gv, true) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + end + return relocs +end + +# Emit a device-resident constant replica of the box holding `obj` and return +# the constant to store in its slot. Any relocatable header is recorded in `relocs`. +function materialize_box!(mod::LLVM.Module, relocs::Relocations, gv::GlobalVariable, + @nospecialize(obj), init::Ptr{Cvoid}) + @assert isbitstype(typeof(obj)) && sizeof(obj) > 0 + + W = sizeof(Int) + hdr, bytes = GC.@preserve obj begin + # the header word transparently yields the smalltag immediate for + # smalltag types and the host type pointer otherwise; drop the gcbits + hdr = unsafe_load(Ptr{UInt}(init - W)) & ~UInt(15) + bytes = [unsafe_load(Ptr{UInt8}(init), i) for i in 1:sizeof(obj)] + hdr, bytes + end + + T_word = LLVM.IntType(8W) + T_byte = LLVM.Int8Type() + patch_header = hdr >= UInt(64 << 4) # jl_max_tags << 4 + fields = LLVM.Constant[ConstantInt(T_word, patch_header ? 0 : hdr), + ConstantDataArray(T_byte, bytes)] + header_idx = 0 + payload_idx = 1 + if Base.datatype_alignment(typeof(obj)) > W + # pad so the payload lands at a 16-byte offset (JL_HEAP_ALIGNMENT max) + pushfirst!(fields, ConstantDataArray(T_byte, zeros(UInt8, 16 - W))) + header_idx = 1 + payload_idx = 2 + end + boxinit = ConstantStruct(fields) + boxty = value_type(boxinit) + + box_name = if patch_header + safe_name(LLVM.name(gv)) * "_" * string(objectid(obj); base=16) * "_box" + else + safe_name(LLVM.name(gv)) * "_box" + end + box = GlobalVariable(mod, boxty, box_name) + LLVM.name(box) == box_name || error("Interior relocation global '$box_name' is already in use") + initializer!(box, boxinit) + alignment!(box, 16) + if patch_header + constant!(box, false) + linkage!(box, LLVM.API.LLVMExternalLinkage) + extinit!(box, true) + offset = Int(offsetof(datalayout(mod), boxty, header_idx)) + relocs.sites[RelocationSite(box_name, offset)] = JuliaValueRef(typeof(obj)) + else + constant!(box, true) + linkage!(box, LLVM.API.LLVMPrivateLinkage) + unnamed_addr!(box, true) + end + + idx(i) = ConstantInt(LLVM.Int32Type(), i) + payload = const_gep(boxty, box, LLVM.Constant[idx(0), idx(payload_idx)]) + slotty = global_value_type(gv) + val = value_type(payload) == slotty ? payload : const_addrspacecast(payload, slotty) + return val +end + +# Some Julia code loads words from libjulia C globals, for example type tags. Record those +# loads as dedicated zero-offset relocations immediately before object emission. +function is_cglobal_candidate(value, relocs::Relocations) + name = LLVM.name(value) + value isa LLVM.GlobalVariable && + haskey(relocs.sites, RelocationSite(name, 0)) && return false + isdeclaration(value) || return false + value isa LLVM.Function && LLVM.isintrinsic(value) && return false + return startswith(name, "jl_") +end + +function collect_cglobal_relocations!(mod::LLVM.Module, relocs::Relocations) + changed = false + + for f in [collect(functions(mod)); collect(globals(mod))] + is_cglobal_candidate(f, relocs) || continue + fn = LLVM.name(f) + slot = nothing + function cglobal_slot() + if slot === nothing + name = "gpu_" * fn + slot = GlobalVariable(mod, relocation_word_type(), name) + LLVM.name(slot) == name || + error("cglobal slot name '$name' is already in use") + relocs.sites[RelocationSite(name, 0)] = CGlobalRef(Symbol(fn)) + end + slot + end + + function replace_bindings!(value) + changed = false + for use in collect(uses(value)) + val = user(use) + if isa(val, LLVM.ConstantExpr) + # recurse + changed |= replace_bindings!(val) + elseif isa(val, LLVM.LoadInst) + T = value_type(val) + if !(T isa LLVM.PointerType || + (T isa LLVM.IntegerType && width(T) == 8sizeof(UInt))) + error("Unsupported cglobal '$fn' load of LLVM type $T") + end + @dispose builder=IRBuilder() begin + position!(builder, val) + replacement = load!(builder, relocation_word_type(), + cglobal_slot()) + if T isa LLVM.PointerType + replacement = inttoptr!(builder, replacement, T) + end + replace_uses!(val, replacement) + end + erase!(val) + changed = true + end + end + changed + end + + changed |= replace_bindings!(f) + end + + return changed +end + +function has_unresolved_cglobal_loads(mod::LLVM.Module, relocs::Relocations) + function has_load(value) + for use in uses(value) + val = user(use) + val isa LLVM.LoadInst && return true + val isa LLVM.ConstantExpr && has_load(val) && return true + end + return false + end + + for value in [collect(functions(mod)); collect(globals(mod))] + is_cglobal_candidate(value, relocs) || continue + has_load(value) && return true + end + return false +end + + +## bookkeeping + +function link_relocatable!(dest_mod::LLVM.Module, dest_relocs::Relocations, + src_mod::LLVM.Module, src_relocs::Relocations; + only_needed=false) + # Make an identical source definition a declaration so LLVM can resolve it to the + # destination while linking. Declaration sites already merge without modification. + for (site, ref) in src_relocs.sites + existing = get(dest_relocs.sites, site, nothing) + existing === nothing && continue + name = site.name + offset = site.offset + same_relocation_target(existing, ref) || + error("Relocation '$name+$offset' refers to conflicting values") + haskey(globals(src_mod), name) || + error("Missing source relocation global '$name'") + gv = globals(src_mod)[name] + isdeclaration(gv) && continue + haskey(globals(dest_mod), name) || + error("Missing destination relocation global '$name'") + initializer!(gv, nothing) + extinit!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + end + + link!(dest_mod, src_mod; only_needed) + for (site, ref) in src_relocs.sites + name = site.name + # A site absent from the linked module was dead (DCE'd or not imported under + # `only_needed`); its relocation dies with it. + haskey(globals(dest_mod), name) || continue + existing = get(dest_relocs.sites, site, nothing) + existing === nothing || same_relocation_target(existing, ref) || + error("Relocation '$(site.name)+$(site.offset)' refers to conflicting values") + dest_relocs.sites[site] = ref + end + return +end + +function prune_dead_relocations!(mod::LLVM.Module, relocs::Relocations) + mod_gvs = globals(mod) + dead_names = Set{String}() + for site in keys(relocs.sites) + gv = haskey(mod_gvs, site.name) ? mod_gvs[site.name] : nothing + if gv === nothing || (!isdeclaration(gv) && isempty(uses(gv))) + push!(dead_names, site.name) + end + end + for site in collect(keys(relocs.sites)) + site.name in dead_names && delete!(relocs.sites, site) + end + for name in dead_names + gv = haskey(mod_gvs, name) ? mod_gvs[name] : nothing + gv === nothing || isdeclaration(gv) || erase!(gv) + end + return +end + + +## lowering + +# Lower live relocations before object emission, dispatching on the back-end's +# `relocation_lowering` strategy. Internal: back-ends select a strategy through the trait +# rather than overriding this. +function lower_relocations!(@nospecialize(job::CompilerJob), mod::LLVM.Module, + relocs::Relocations) + strategy = relocation_lowering(job) + if strategy === :bake + bake_relocations!(mod, relocs) + elseif strategy === :patch + emit_patchable_relocations!(mod, relocs) + elseif strategy === :import + emit_imported_relocations!(mod, relocs) + elseif strategy === :defer + # the consumer resolves relocations itself at the `:llvm` level; emitting an + # object here would leave its sites permanently unresolved + isempty(relocs.sites) || + error("Job defers relocation lowering to its consumer, so code with live " * + "relocations cannot be emitted. Use `apply_relocations!` on the " * + "`:llvm` result, or a `:patch`/`:import` lowering strategy.") + else + error("Unknown relocation lowering strategy :$strategy") + end + return +end + +""" + bake_relocations!(mod, relocs) + +Resolve every site in the current Julia process and write the resulting words into the IR, +leaving `relocs` empty. The module then embeds session-local addresses and must not be +persisted across sessions. Drop dead sites first with [`prune_dead_relocations!`](@ref). +""" +function bake_relocations!(mod::LLVM.Module, relocs::Relocations) + foreach_relocation(mod, relocs) do site, gv, ref + if isdeclaration(gv) + value = resolve_relocation_target(ref) + val = slot_initializer(gv, value) + initializer!(gv, val) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + constant!(gv, true) + else + init = initializer(gv) + T = value_type(init)::LLVM.StructType + idx = Int(element_at(datalayout(mod), T, site.offset)) + 1 + # An all-zero box (e.g. a patchable header over a zero payload) is folded to a + # `zeroinitializer`, a `ConstantAggregateZero` that reports no operands; rebuild + # the explicit per-field constants from the struct's element types. + fields = if init isa LLVM.ConstantAggregateZero + LLVM.Constant[null(elty) for elty in elements(T)] + else + LLVM.Constant[operands(init)...] + end + fields[idx] = ConstantInt(value_type(fields[idx]), resolve_relocation_target(ref)) + initializer!(gv, ConstantStruct(T, fields)) + linkage!(gv, LLVM.API.LLVMPrivateLinkage) + extinit!(gv, false) + constant!(gv, true) + unnamed_addr!(gv, true) + end + end + empty!(relocs.sites) + return +end + +""" + emit_patchable_relocations!(mod, relocs) + +Emit declarations as writable, null-initialized definitions. The loader must patch every +relocation after loading the object. This requires a per-object symbol namespace. +""" +function emit_patchable_relocations!(mod::LLVM.Module, relocs::Relocations) + used = GlobalVariable[] + foreach_relocation(mod, relocs) do _, gv, _ + if isdeclaration(gv) + initializer!(gv, null(global_value_type(gv))) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, true) + end + push!(used, gv) + end + isempty(used) || set_used!(mod, used...) + return +end + +""" + emit_imported_relocations!(mod, relocs) + +Leave declaration slots external so a JIT loader resolves them at link time (e.g. ORC +`absoluteSymbols`); a word inside a definition cannot be imported, so those sites stay +`llvm.used` for the loader to patch after loading. Requires a per-object symbol namespace. +""" +function emit_imported_relocations!(mod::LLVM.Module, relocs::Relocations) + used = GlobalVariable[] + foreach_relocation(mod, relocs) do _, gv, _ + if isdeclaration(gv) + constant!(gv, false) + linkage!(gv, LLVM.API.LLVMExternalLinkage) + extinit!(gv, false) + else + push!(used, gv) + end + end + isempty(used) || set_used!(mod, used...) + return +end + +""" + apply_relocations!(mod, relocs) + +Loader entry point for `:defer` back-ends. Resolves every live site into `mod` without +consuming `relocs`, so cached metadata can be reused. Sites whose global was optimized away +are skipped. Resolution permanently roots referenced Julia values in the process. +Apply once per parsed module. +""" +function apply_relocations!(mod::LLVM.Module, relocs::Relocations) + live = copy(relocs) + prune_dead_relocations!(mod, live) + bake_relocations!(mod, live) + return +end + + +## introspection + +function referenced_object(value, relocs::Relocations) + # This is best-effort: optimized shapes fall back to the unknown-binding error path. + while value isa ConstantExpr && + opcode(value) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) + value = first(operands(value)) + end + if value isa LLVM.LoadInst + source = first(operands(value)) + while source isa ConstantExpr && + opcode(source) in (LLVM.API.LLVMBitCast, LLVM.API.LLVMAddrSpaceCast) + source = first(operands(source)) + end + if source isa GlobalVariable + ref = get(relocs.sites, RelocationSite(LLVM.name(source), 0), nothing) + ref isa JuliaValueRef && return Some(ref.value) + end + elseif value isa ConstantExpr && opcode(value) == LLVM.API.LLVMIntToPtr + ptr = Ptr{Cvoid}(convert(Int, first(operands(value)))) + return Some(Base.unsafe_pointer_to_objref(ptr)) + end + return nothing +end diff --git a/src/rtlib.jl b/src/rtlib.jl index b60ba806..d7bdd87b 100644 --- a/src/rtlib.jl +++ b/src/rtlib.jl @@ -60,22 +60,20 @@ end ## functionality to build the runtime library -# Per-function compilation results for the GPU runtime library, cached through the -# same `cached_results` mechanism back-ends use for kernels. On 1.11+ the bitcode -# thus lives on the runtime function's `CodeInstance` — possibly alongside a -# back-end's own results struct — and persists through precompilation, so sessions -# loading a back-end that compiled its runtime during precompile skip codegen -# entirely. On 1.10 it is cached for the duration of the session. +# Per-function relocatable bitcode for the GPU runtime library. When Julia exposes the +# required relocation metadata, this persists with the function's `CodeInstance`. +# `runtime_libs` below provides the session cache on older Julia versions. mutable struct RuntimeFunctionResults bitcode::Union{Nothing,Vector{UInt8}} - RuntimeFunctionResults() = new(nothing) + relocations::Relocations + RuntimeFunctionResults() = new(nothing, Relocations()) end # Compile a single runtime function and link it into `mod`. The renamed bitcode is # memoized through `RuntimeFunctionResults`; the session-local `runtime_libs` cache # below additionally avoids repeating the parse-and-link work within a session. -function emit_function!(mod, config::CompilerConfig, source::MethodInstance, method, - world::UInt) +function emit_function!(mod, relocs::Relocations, config::CompilerConfig, + source::MethodInstance, method, world::UInt) name = method.llvm_name rt_job = CompilerJob(source, config, world) @@ -83,12 +81,16 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met # inference itself. ci, res = runtime_function_results(rt_job) if res !== nothing && res.bitcode !== nothing - link!(mod, parse(LLVM.Module, MemoryBuffer(res.bitcode))) + link_relocatable!(mod, relocs, + parse(LLVM.Module, MemoryBuffer(res.bitcode)), + res.relocations) ci === nothing && (ci = runtime_code_instance(rt_job)) return ci::CodeInstance end - new_mod, meta = compile_unhooked(:llvm, rt_job) + # Keep this intermediate module relocatable even when the final back-end resolves + # relocations eagerly. The caller links a fresh copy and lowers the merged sites. + new_mod, meta = compile_unhooked(:llvm, rt_job; resolve_relocations=false) ft = function_type(meta.entry) expected_ft = convert(LLVM.FunctionType, method) if return_type(ft) != return_type(expected_ft) @@ -97,13 +99,7 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met # recent Julia versions include prototypes for all runtime functions, even if unused run!(StripDeadPrototypesPass(), new_mod, llvm_machine(config.target)) - - # Resolve constgv mappings before their metadata is discarded. Dedicated Bool - # globals are resolved after linking into the toplevel module. - if !isempty(meta.gv_to_value) - portable = relocate_gvs!(new_mod, meta.gv_to_value) - portable || mark_session_dependent!(rt_job) - end + prune_dead_relocations!(new_mod, meta.relocations) # rename to the final `gpu_*` name on the per-function module, so the cached bitcode # is immediately link-ready (no per-session rename pass on a cache hit). @@ -118,17 +114,29 @@ function emit_function!(mod, config::CompilerConfig, source::MethodInstance, met io = IOBuffer() write(io, new_mod) ci === nothing && (ci = runtime_code_instance(rt_job)) - res === nothing && (res = job_results(RuntimeFunctionResults, ci, rt_job.config)) - res.bitcode = take!(io) + if supports_relocatable_ir() + res === nothing && (res = runtime_results(RuntimeFunctionResults, ci, rt_job.config)) + res.bitcode = take!(io) + res.relocations = meta.relocations + end - link!(mod, new_mod) + link_relocatable!(mod, relocs, new_mod, meta.relocations) return ci::CodeInstance end +function runtime_results(::Type{V}, ci::CodeInstance, config::CompilerConfig) where {V} + @static if HAS_INTEGRATED_CACHE + if supports_relocatable_ir() + return persistent_results(V, ci, config) + end + end + return session_results(V, ci, config) +end + function runtime_function_results(@nospecialize(job::CompilerJob)) ci = job_code_instance(job) ci === nothing && return nothing, nothing - return ci, job_results(RuntimeFunctionResults, ci, job.config) + return ci, runtime_results(RuntimeFunctionResults, ci, job.config) end function runtime_method_instance(@nospecialize(job::CompilerJob), method) @@ -170,13 +178,14 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) mod = LLVM.Module("GPUCompiler run-time library") sources = MethodInstance[] code_instances = CodeInstance[] + relocs = Relocations() for method in values(Runtime.methods) resolved = runtime_method_instance(job, method) resolved === nothing && continue source = resolved push!(sources, source) - push!(code_instances, emit_function!(mod, config, source, method, job.world)) + push!(code_instances, emit_function!(mod, relocs, config, source, method, job.world)) end # we cannot optimize the runtime library, because the code would then be optimized again @@ -184,13 +193,13 @@ function build_runtime(@nospecialize(job::CompilerJob), config::CompilerConfig) # removes Julia address spaces, which would then lead to type mismatches when using # functions from the runtime library from IR that has not been stripped of AS info. - return mod, sources, code_instances + return mod, sources, code_instances, relocs end # Session-local cache of assembled runtime libraries, keyed by # `(runtime_config(job), opaque_pointers)`: the derived runtime config covers every -# codegen-relevant setting (e.g. the debug level, which is baked into the runtime IR -# as a constant), while cosmetic kernel-job fields are normalized away. Cross-session +# codegen-relevant setting (e.g. the debug level stored in the runtime IR), while +# cosmetic kernel-job fields are normalized away. Cross-session # persistence happens at the per-function level (see `RuntimeFunctionResults`): # reassemble on first use of each session, then reuse within the session. # @@ -203,6 +212,7 @@ mutable struct RuntimeLibrary sources::Vector{MethodInstance} code_instances::Vector{CodeInstance} validated_world::UInt + relocations::Relocations end function runtime_library_valid(lib::RuntimeLibrary, @nospecialize(job::CompilerJob)) @@ -235,14 +245,16 @@ const runtime_libs_lock = ReentrantLock() cached = Base.@lock runtime_libs_lock begin cached = get(runtime_libs, key, nothing) if cached === nothing || !runtime_library_valid(cached, job) - lib, sources, code_instances = build_runtime(job, config) + lib, sources, code_instances, relocations = build_runtime(job, config) io = IOBuffer() write(io, lib) - cached = RuntimeLibrary(take!(io), sources, code_instances, job.world) + cached = RuntimeLibrary(take!(io), sources, code_instances, job.world, + relocations) runtime_libs[key] = cached end cached end - return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true) + return parse(LLVM.Module, MemoryBuffer(cached.bytes); lazy=true), + cached.relocations end diff --git a/src/utils.jl b/src/utils.jl index db2fc7d7..cae25a8e 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -418,3 +418,14 @@ end return inits end end + +"""Whether Julia exposes enough global-variable metadata to emit relocatable IR.""" +supports_relocatable_ir() = @static if VERSION >= v"1.13.0-DEV.623" + true +else + # `jl_get_llvm_gvs_globals` was backported to 1.10, so the symbol alone is not enough: + # 1.10's codegen still embeds Julia addresses (as `inttoptr` constants) in the JIT + # (non-imaging) mode we compile in, instead of emitting the relocatable global + # declarations the relocation machinery collects. Only 1.11+ emits those declarations. + VERSION >= v"1.11-" && HAS_LLVM_GVS_GLOBALS +end diff --git a/src/validation.jl b/src/validation.jl index c8e97638..0e19d440 100644 --- a/src/validation.jl +++ b/src/validation.jl @@ -162,8 +162,8 @@ end # `show` via `showerror`, avoiding the default field-dump that derefs disposed IR Base.show(io::IO, err::InvalidIRError) = showerror(io, err) -function check_ir(job, args...) - errors = check_ir!(job, IRError[], args...) +function check_ir(job, mod::LLVM.Module, relocs::Relocations=Relocations()) + errors = check_ir!(job, IRError[], mod, relocs) unique!(errors) if !isempty(errors) throw(InvalidIRError(job, errors)) @@ -172,9 +172,9 @@ function check_ir(job, args...) return end -function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module) +function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module, relocs::Relocations) for f in functions(mod) - check_ir!(job, errors, f) + check_ir!(job, errors, f, relocs) end # custom validation @@ -183,10 +183,10 @@ function check_ir!(job, errors::Vector{IRError}, mod::LLVM.Module) return errors end -function check_ir!(job, errors::Vector{IRError}, f::LLVM.Function) +function check_ir!(job, errors::Vector{IRError}, f::LLVM.Function, relocs::Relocations) for bb in blocks(f), inst in instructions(bb) if isa(inst, LLVM.CallInst) - check_ir!(job, errors, inst) + check_ir!(job, errors, inst, relocs) elseif isa(inst, LLVM.LoadInst) check_ir!(job, errors, inst) end @@ -221,7 +221,7 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.LoadInst) return errors end -function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) +function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst, relocs::Relocations) bt = backtrace(inst) dest = called_operand(inst) if isa(dest, LLVM.Function) @@ -233,11 +233,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) elseif fn == "jl_get_binding_or_error" || fn == "ijl_get_binding_or_error" try m, sym = arguments(inst) - sym = first(operands(sym::ConstantExpr))::ConstantInt - sym = convert(Int, sym) - sym = Ptr{Cvoid}(sym) - sym = Base.unsafe_pointer_to_objref(sym) - push!(errors, (DELAYED_BINDING, bt, sym)) + ref = referenced_object(sym, relocs) + ref === nothing && error("Unknown binding") + push!(errors, (DELAYED_BINDING, bt, something(ref))) catch e @safe_debug "Decoding arguments to jl_get_binding_or_error failed" inst bb=LLVM.parent(inst) push!(errors, (DELAYED_BINDING, bt, nothing)) @@ -246,10 +244,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) fn == "jl_get_binding_value_seqcst" || fn == "ijl_get_binding_value_seqcst" try # pry the binding from the IR - expr = arguments(inst)[1]::ConstantExpr - expr = first(operands(expr))::ConstantInt # get rid of inttoptr - ptr = Ptr{Any}(convert(Int, expr)) - obj = Base.unsafe_pointer_to_objref(ptr) + ref = referenced_object(arguments(inst)[1], relocs) + ref === nothing && error("Unknown binding") + obj = something(ref) push!(errors, (DELAYED_BINDING, bt, obj.globalref)) catch e @safe_debug "Decoding arguments to jl_reresolve_binding_value_seqcst failed" inst bb=LLVM.parent(inst) @@ -258,10 +255,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) elseif startswith(fn, "tojlinvoke") try fun, args, nargs = arguments(inst) - fun = first(operands(fun::ConstantExpr))::ConstantInt - fun = convert(Int, fun) - fun = Ptr{Cvoid}(fun) - fun = Base.unsafe_pointer_to_objref(fun)::Base.Function + ref = referenced_object(fun, relocs) + ref === nothing && error("Unknown function") + fun = something(ref)::Base.Function push!(errors, (DYNAMIC_CALL, bt, fun)) # XXX: an invoke trampoline happens when codegen doesn't have access to code # which suggests a GPUCompiler.jl bug. throw an error instead? @@ -279,10 +275,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) end try fun, args, nargs, meth = arguments(inst) - meth = first(operands(meth::ConstantExpr))::ConstantInt - meth = convert(Int, meth) - meth = Ptr{Cvoid}(meth) - meth = Base.unsafe_pointer_to_objref(meth)::Core.MethodInstance + ref = referenced_object(meth, relocs) + ref === nothing && error("Unknown method instance") + meth = something(ref)::Core.MethodInstance push!(errors, (DYNAMIC_CALL, bt, meth.def)) catch e @safe_debug "Decoding arguments to jl_invoke failed" inst bb=LLVM.parent(inst) @@ -291,10 +286,9 @@ function check_ir!(job, errors::Vector{IRError}, inst::LLVM.CallInst) elseif fn == "jl_apply_generic" || fn == "ijl_apply_generic" try f, args, nargs = arguments(inst) - f = first(operands(f))::ConstantInt # get rid of inttoptr - f = convert(Int, f) - f = Ptr{Cvoid}(f) - f = Base.unsafe_pointer_to_objref(f) + ref = referenced_object(f, relocs) + ref === nothing && error("Unknown function") + f = something(ref) push!(errors, (DYNAMIC_CALL, bt, f)) catch e @safe_debug "Decoding arguments to jl_apply_generic failed" inst bb=LLVM.parent(inst) diff --git a/test/helpers/native.jl b/test/helpers/native.jl index 63d8a0f3..e8fa597d 100644 --- a/test/helpers/native.jl +++ b/test/helpers/native.jl @@ -1,6 +1,7 @@ module Native using ..GPUCompiler +using LLVM import ..TestRuntime # local method table for device functions @@ -9,9 +10,13 @@ Base.Experimental.@MethodTable(test_method_table) struct CompilerParams <: AbstractCompilerParams entry_safepoint::Bool method_table + jit::Bool + patch::Bool + defer::Bool - CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table) = - new(entry_safepoint, method_table) + CompilerParams(entry_safepoint::Bool=false, method_table=test_method_table, + jit::Bool=false, patch::Bool=false, defer::Bool=false) = + new(entry_safepoint, method_table, jit, patch, defer) end module Runtime end @@ -22,16 +27,75 @@ GPUCompiler.runtime_module(::NativeCompilerJob) = Runtime GPUCompiler.method_table(@nospecialize(job::NativeCompilerJob)) = job.config.params.method_table GPUCompiler.can_safepoint(@nospecialize(job::NativeCompilerJob)) = job.config.params.entry_safepoint +# Object-emitting modes drive an ORC loader (see `load`): `jit` imports declarations, +# while `patch` emits definitions to write after loading. `defer` stops at `:llvm`. +GPUCompiler.relocation_lowering(@nospecialize(job::NativeCompilerJob)) = + job.config.params.defer ? :defer : + job.config.params.patch ? :patch : + job.config.params.jit ? :import : :bake + +function GPUCompiler.mcgen(@nospecialize(job::NativeCompilerJob), mod::LLVM.Module, + format=LLVM.API.LLVMAssemblyFile) + if job.config.params.jit || job.config.params.patch + target = job.config.target + @dispose tm=JITTargetMachine(GPUCompiler.llvm_triple(target), target.cpu, + target.features) begin + return String(emit(tm, mod, format)) + end + else + return invoke(GPUCompiler.mcgen, Tuple{CompilerJob,LLVM.Module,Any}, + job, mod, format) + end +end + function create_job(@nospecialize(func), @nospecialize(types); - entry_safepoint::Bool=false, method_table=test_method_table, kwargs...) + entry_safepoint::Bool=false, method_table=test_method_table, + jit::Bool=false, patch::Bool=false, defer::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = NativeCompilerTarget(;jlruntime=true) - params = CompilerParams(entry_safepoint, method_table) + params = CompilerParams(entry_safepoint, method_table, jit, patch, defer) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end +function load(obj::Vector{UInt8}, entry::String, relocs::GPUCompiler.Relocations, + ir::LLVM.Module) + lljit = LLJIT(; tm=JITTargetMachine()) + try + jd = JITDylib(lljit) + prefix = LLVM.get_prefix(lljit) + add!(jd, LLVM.CreateDynamicLibrarySearchGeneratorForProcess(prefix)) + + relocations = GPUCompiler.resolved_relocations(relocs) + declarations = [(site, value) for (site, value) in relocations + if isdeclaration(globals(ir)[site.name])] + cells = Vector{UInt}(undef, length(declarations)) + pairs = LLVM.API.LLVMOrcCSymbolMapPair[] + for (i, (site, value)) in enumerate(declarations) + cells[i] = value + symbol = LLVM.API.LLVMJITEvaluatedSymbol( + reinterpret(UInt, pointer(cells, i)), + LLVM.API.LLVMJITSymbolFlags( + LLVM.API.LLVMJITSymbolGenericFlagsExported, 0)) + push!(pairs, LLVM.API.LLVMOrcCSymbolMapPair(mangle(lljit, site.name), symbol)) + end + isempty(pairs) || LLVM.define(jd, LLVM.absolute_symbols(pairs)) + + add!(lljit, jd, MemoryBuffer(obj)) + for (site, value) in relocations + isdeclaration(globals(ir)[site.name]) && continue + addr = lookup(lljit, site.name) + unsafe_store!(Ptr{UInt}(pointer(addr) + site.offset), value) + end + addr = lookup(lljit, entry) + return pointer(addr), (lljit, cells) + catch + dispose(lljit) + rethrow() + end +end + function code_typed(@nospecialize(func), @nospecialize(types); kwargs...) job, kwargs = create_job(func, types; kwargs...) GPUCompiler.code_typed(job; kwargs...) diff --git a/test/helpers/ptx.jl b/test/helpers/ptx.jl index 4b00f707..d2233325 100644 --- a/test/helpers/ptx.jl +++ b/test/helpers/ptx.jl @@ -3,10 +3,17 @@ module PTX using ..GPUCompiler import ..TestRuntime -struct CompilerParams <: AbstractCompilerParams end +struct CompilerParams <: AbstractCompilerParams + patch::Bool + CompilerParams(patch::Bool=false) = new(patch) +end PTXCompilerJob = CompilerJob{PTXCompilerTarget,CompilerParams} +# `patch=true` keeps relocations symbolic (as CUDA.jl does); plain jobs resolve them in IR. +GPUCompiler.relocation_lowering(@nospecialize(job::PTXCompilerJob)) = + job.config.params.patch ? :patch : :bake + struct PTXKernelState data::Int64 end @@ -39,14 +46,14 @@ function create_job(@nospecialize(func), @nospecialize(types); cap=v"7.0", ptx=v"6.0", feature_set=:baseline, minthreads=nothing, maxthreads=nothing, blocks_per_sm=nothing, maxregs=nothing, - fastmath=false, + fastmath=false, patch::Bool=false, kwargs...) config_kwargs, kwargs = split_kwargs(kwargs, GPUCompiler.CONFIG_KWARGS) source = methodinstance(typeof(func), Base.to_tuple_type(types), Base.get_world_counter()) target = PTXCompilerTarget(; cap, ptx, feature_set, minthreads, maxthreads, blocks_per_sm, maxregs, fastmath) - params = CompilerParams() + params = CompilerParams(patch) config = CompilerConfig(target, params; kernel=false, config_kwargs...) CompilerJob(source, config), kwargs end diff --git a/test/native.jl b/test/native.jl index 834c0f20..2b50a2cd 100644 --- a/test/native.jl +++ b/test/native.jl @@ -59,7 +59,8 @@ end end end - job, _ = Native.create_job(mod.outer, (Int, Symbol); validate=false) + # A JIT back-end keeps the Symbol reference symbolic in `:llvm`. + job, _ = Native.create_job(mod.outer, (Int, Symbol); validate=false, jit=true) JuliaContext() do ctx ir, meta = GPUCompiler.compile(:llvm, job) @@ -72,18 +73,10 @@ end @test length(other_mis) == 1 @test only(other_mis).def in methods(mod.inner) - if VERSION >= v"1.12" - @test length(meta.gv_to_value) == 1 - for (k, v) in meta.gv_to_value - @test v != C_NULL - end + if GPUCompiler.supports_relocatable_ir() + @test length(meta.relocations.sites) == 1 + @test only(values(meta.relocations.sites)) isa GPUCompiler.JuliaValueRef end - # TODO: Global values get privatized, so we can't find them by name anymore. - # %.not = icmp eq ptr %"sym::Symbol", inttoptr (i64 140096668482288 to ptr), !dbg !38 - # for (name, v) in meta.gv_to_value - # gv = globals(ir)[name] - # @test LLVM.initializer(gv) === v - # end end end @@ -225,21 +218,6 @@ end @test new_res !== res @test new_res.asm === nothing - @static if GPUCompiler.HAS_INTEGRATED_CACHE - # session-dependent results (e.g. artifacts with relocated GVs) are wiped - # before image serialization; emulate the atexit-driven wipe directly - new_res.asm = "session-dependent" - other_job, _ = Native.create_job(mod.kernel, (Int64,); name="other") - other_res = GPUCompiler.cached_results(mod.Results, other_job) - push!(GPUCompiler.session_dependent_jobs, new_job) - GPUCompiler.wipe_session_dependent_results() - @test isempty(GPUCompiler.session_dependent_jobs) - wiped_res = GPUCompiler.cached_results(mod.Results, new_job) - @test wiped_res !== new_res - @test wiped_res.asm === nothing - # ... without affecting other configs on the same CI - @test GPUCompiler.cached_results(mod.Results, other_job) === other_res - end end @testset "runtime cache invalidation" begin @@ -273,11 +251,10 @@ end end end - @testset "runtime constgv relocation" begin + @testset "runtime relocations" begin # runtime functions like `box_bool` may reference Julia singletons through - # `julia.constgv` globals. Their session-absolute addresses must be baked into - # the cached runtime bitcode when it is built: only kernel modules go through - # `relocate_gvs!`, so a slot left null here would stay null on the device. + # `julia.constgv` globals. Keep their Julia identities with the cached bitcode + # so the final kernel can resolve them in its own session. job, _ = Native.create_job(identity, (Nothing,)) JuliaContext() do ctx GPUCompiler.load_runtime(job) @@ -292,10 +269,12 @@ end isempty(uses(gv)) && continue used += 1 init = LLVM.initializer(gv) - @test init !== nothing && !LLVM.isnull(init) + @test init === nothing + site = GPUCompiler.RelocationSite(LLVM.name(gv), 0) + @test haskey(lib.relocations.sites, site) end - @static if VERSION >= v"1.12-" - # on older versions, Julia bakes addresses without tagging globals + if GPUCompiler.supports_relocatable_ir() + # otherwise Julia embeds addresses without tagging globals @test used > 0 end end @@ -369,7 +348,8 @@ end Native.code_execution(mod.kernel, (Ptr{Int64}, Bool, Int32)) Native.code_execution(mod.egal_kernel, (Ptr{Bool}, Bool, Int32)) - # relocate_gvs! reports whether the module stayed session-portable + # Classification records whether the module stayed session-portable; eager + # lowering then resolves any remaining relocation slots. JuliaContext() do ctx # Unlike Int128, vector-shaped tuples are 16-byte aligned on all # supported architectures and Julia versions. @@ -392,7 +372,10 @@ end gv = LLVM.GlobalVariable(m, LLVM.PointerType(LLVM.Int8Type()), name) constant!(gv, true) end - @test GPUCompiler.relocate_gvs!(m, Dict{String, Ptr{Cvoid}}()) + relocs = GPUCompiler.collect_julia_value_relocations!( + m, Dict{String, Ptr{Cvoid}}()) + @test isempty(relocs.sites) + GPUCompiler.bake_relocations!(m, relocs) bool_ir = string(m) for name in ("jl_true", "jl_false") @test haskey(globals(m), "$(name)_box") @@ -405,27 +388,51 @@ end GC.@preserve objs begin # smalltag isbits: materialized, portable m, map = slot_module(ptrs[1]) - @test GPUCompiler.relocate_gvs!(m, map) + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + @test isempty(relocs.sites) + GPUCompiler.bake_relocations!(m, relocs) @test haskey(globals(m), "jl_global_0_box") dispose(m) - # Float64: materialized, but the header carries a type pointer + # Float64: the non-smalltag header is an interior relocation. m, map = slot_module(ptrs[2]) - @test !GPUCompiler.relocate_gvs!(m, map) - @test haskey(globals(m), "jl_global_0_box") + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + @test length(relocs.sites) == 1 + site, ref = only(relocs.sites) + @test site.offset == 0 + @test ref.value === Float64 + box = globals(m)[site.name] + @test isextinit(box) + @test linkage(box) == LLVM.API.LLVMExternalLinkage + header_idx = Int(element_at(datalayout(m), global_value_type(box), + site.offset)) + 1 + @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == 0 + GPUCompiler.bake_relocations!(m, relocs) + @test isempty(relocs.sites) + @test !isextinit(box) + @test isconstant(box) + @test linkage(box) == LLVM.API.LLVMPrivateLinkage + @test convert(UInt, collect(operands(initializer(box)))[header_idx]) == + GPUCompiler.resolve_relocation_target(ref) dispose(m) - # Symbol: baked address + # Symbol: resolved address m, map = slot_module(ptrs[3]) - @test !GPUCompiler.relocate_gvs!(m, map) + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + @test only(values(relocs.sites)).value === objs[3] + GPUCompiler.bake_relocations!(m, relocs) + @test isempty(relocs.sites) @test !haskey(globals(m), "jl_global_0_box") @test occursin("inttoptr", string(m)) dispose(m) # 16-byte-aligned payloads get padded past the header word m, map = slot_module(ptrs[4]) - GPUCompiler.relocate_gvs!(m, map) - box = globals(m)["jl_global_0_box"] + relocs = GPUCompiler.collect_julia_value_relocations!(m, map) + site, _ = only(relocs.sites) + @test site.offset == 8 + GPUCompiler.bake_relocations!(m, relocs) + box = globals(m)[site.name] @test length(elements(LLVM.global_value_type(box))) == 3 dispose(m) end @@ -731,12 +738,264 @@ end end end -@testset "CPU reference resolution" begin +@testset "relocation target resolution" begin + @test_throws ArgumentError GPUCompiler.RelocationSite("invalid", -1) + + sym = :relocation_target_probe + @test GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(sym)) == + UInt(pointer_from_objref(sym)) + + singleton = nothing + @test GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(singleton)) == + UInt(ccall(:jl_value_ptr, Ptr{Cvoid}, (Any,), singleton)) + + @test_throws ErrorException GPUCompiler.JuliaValueRef(1.5) +end + +@testset "postponed relocation" begin + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + f() = UInt(pointer_from_objref(:relocation_probe)) + end + job, _ = Native.create_job(mod.f, Tuple{}; jit=true) + JuliaContext() do ctx + obj, meta = GPUCompiler.compile(:obj, job) + relocs = meta.relocations + @test !isempty(relocs.sites) + @test all(ref -> ref isa GPUCompiler.JuliaValueRef, values(relocs.sites)) + @test all(site -> isdeclaration(globals(meta.ir)[site.name]), keys(relocs.sites)) + + bytes = Vector{UInt8}(codeunits(obj)) + entry = LLVM.name(meta.entry) + probe_site = findfirst(ref -> ref isa GPUCompiler.JuliaValueRef && + ref.value === :relocation_probe, relocs.sites) + @test probe_site !== nothing + expected = GPUCompiler.resolve_relocation_target(relocs.sites[probe_site]) + fptr, keepalive = Native.load(bytes, entry, relocs, meta.ir) + try + GC.@preserve keepalive begin + actual = ccall(fptr, UInt, ()) + @test actual == expected + end + finally + dispose(first(keepalive)) + end + + @test_throws LLVMException Native.load(bytes, entry, GPUCompiler.Relocations(), + meta.ir) + + # Sites in definitions are applied after the object has been loaded. + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + patch_mod = parse(LLVM.Module, """ + @patched_box = externally_initialized global { i64, i64 } { i64 0, i64 42 } + + define i64 @patched_header() { + %value = load i64, $(ptr("i64")) getelementptr ({ i64, i64 }, $(ptr("{ i64, i64 }")) @patched_box, i32 0, i32 0) + ret i64 %value + }""") + triple!(patch_mod, GPUCompiler.llvm_triple(job.config.target)) + datalayout!(patch_mod, GPUCompiler.llvm_datalayout(job.config.target)) + patch_relocs = GPUCompiler.Relocations() + site = GPUCompiler.RelocationSite("patched_box", 0) + patch_relocs.sites[site] = GPUCompiler.JuliaValueRef(Float64) + patch_obj = GPUCompiler.mcgen(job, patch_mod, LLVM.API.LLVMObjectFile) + patch_ptr, patch_keepalive = Native.load( + Vector{UInt8}(codeunits(patch_obj)), "patched_header", patch_relocs, + patch_mod) + try + GC.@preserve patch_keepalive begin + @test ccall(patch_ptr, UInt, ()) == + GPUCompiler.resolve_relocation_target( + GPUCompiler.JuliaValueRef(Float64)) + end + finally + dispose(first(patch_keepalive)) + end + end + end +end + +@testset "eager relocation resolution" begin + # Eager resolution in `emit_llvm` leaves nothing for a loader. + mod = @eval module $(gensym()) + probe() = UInt(pointer_from_objref(:eager_probe)) + end + job, _ = Native.create_job(mod.probe, Tuple{}) + JuliaContext() do ctx + ir, meta = GPUCompiler.compile(:llvm, job) + @test isempty(meta.relocations.sites) + # nothing is left for a loader to patch or import + @test !any(GPUCompiler.isextinit, globals(ir)) + + # This back-end can emit objects without threading relocation metadata. + code, _ = GPUCompiler.emit_asm(job, ir, LLVM.API.LLVMObjectFile) + @test !isempty(code) + end +end + +@testset "patchable relocation" begin + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + f() = UInt(pointer_from_objref(:patch_probe)) + end + job, _ = Native.create_job(mod.f, Tuple{}; patch=true) + JuliaContext() do ctx + obj, meta = GPUCompiler.compile(:obj, job) + relocs = meta.relocations + @test !isempty(relocs.sites) + + # every declaration slot became a null-init, externally-initialized definition + # kept alive by `llvm.used`; the loader patches each site after loading. + @test haskey(globals(meta.ir), "llvm.used") + for site in keys(relocs.sites) + gv = globals(meta.ir)[site.name] + @test !isdeclaration(gv) + @test isextinit(gv) + @test !isconstant(gv) + @test linkage(gv) == LLVM.API.LLVMExternalLinkage + @test LLVM.isnull(initializer(gv)) + end + + bytes = Vector{UInt8}(codeunits(obj)) + entry = LLVM.name(meta.entry) + probe_site = findfirst(ref -> ref isa GPUCompiler.JuliaValueRef && + ref.value === :patch_probe, relocs.sites) + @test probe_site !== nothing + expected = GPUCompiler.resolve_relocation_target(relocs.sites[probe_site]) + fptr, keepalive = Native.load(bytes, entry, relocs, meta.ir) + try + GC.@preserve keepalive begin + @test ccall(fptr, UInt, ()) == expected + end + finally + dispose(first(keepalive)) + end + end + end +end + +@testset "deferred relocation" begin + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + f() = UInt(pointer_from_objref(:defer_probe)) + end + job, _ = Native.create_job(mod.f, Tuple{}; defer=true) + JuliaContext() do ctx + ir, meta = GPUCompiler.compile(:llvm, job) + relocs = meta.relocations + @test !isempty(relocs.sites) + + # a deferring consumer caches the bitcode and the relocation metadata... + bitcode = let io = IOBuffer() + write(io, ir) + take!(io) + end + + # ...and every session applies the metadata to a freshly parsed module + for _ in 1:2 + session_mod = parse(LLVM.Module, MemoryBuffer(bitcode)) + GPUCompiler.apply_relocations!(session_mod, relocs) + @test !isempty(relocs.sites) # the metadata is not consumed + for site in keys(relocs.sites) + @test !isdeclaration(globals(session_mod)[site.name]) + end + end + + # emitting an object while relocations are live is a consumer error + @test_throws "defers relocation lowering" GPUCompiler.compile(:obj, job) + end + end +end + +@testset "relocation validation errors" begin + JuliaContext() do ctx + word() = GPUCompiler.relocation_word_type() + nop = (_site, _gv, _ref) -> nothing + ref = GPUCompiler.JuliaValueRef(:probe) + reloc(name, offset=0) = + GPUCompiler.Relocations(Dict(GPUCompiler.RelocationSite(name, offset) => ref)) + + # a site whose global is absent from the module + mod = LLVM.Module("errors") + @test_throws "Missing relocation global" GPUCompiler.foreach_relocation( + nop, mod, reloc("absent")) + + # a declaration slot may only carry a zero offset + mod = LLVM.Module("errors") + GlobalVariable(mod, word(), "slot") + @test_throws "nonzero offset" GPUCompiler.foreach_relocation( + nop, mod, reloc("slot", 8)) + + # a declaration slot must be word-sized + mod = LLVM.Module("errors") + GlobalVariable(mod, LLVM.Int32Type(), "narrow") + @test_throws "has size" GPUCompiler.foreach_relocation(nop, mod, reloc("narrow")) + + # an interior site must land within its global + mod = LLVM.Module("errors") + gv = GlobalVariable(mod, LLVM.StructType([LLVM.Int64Type(), LLVM.Int64Type()]), "box") + initializer!(gv, ConstantStruct(LLVM.Constant[ConstantInt(0), ConstantInt(0)])) + @test_throws "outside its" GPUCompiler.foreach_relocation(nop, mod, reloc("box", 16)) + end +end + +@testset "prune dead relocations" begin + JuliaContext() do ctx + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + mod = parse(LLVM.Module, """ + @live = external global i64 + @dead = internal global { i64, i64 } { i64 0, i64 0 } + define i64 @use() { + %v = load i64, $(ptr("i64")) @live + ret i64 %v + }""") + site(name, offset=0) = GPUCompiler.RelocationSite(name, offset) + relocs = GPUCompiler.Relocations(Dict( + site("live") => GPUCompiler.JuliaValueRef(:live), + site("dead") => GPUCompiler.JuliaValueRef(:dead), # unused definition + site("absent") => GPUCompiler.JuliaValueRef(:absent), # global already gone + )) + GPUCompiler.prune_dead_relocations!(mod, relocs) + @test collect(keys(relocs.sites)) == [site("live")] + @test haskey(globals(mod), "live") # a used declaration survives + @test !haskey(globals(mod), "dead") # the dead definition is erased + end +end + +@testset "resolve zeroinitializer box" begin + # An all-zero box (a patchable header over a zero payload) is folded by LLVM to a + # `zeroinitializer`, a ConstantAggregateZero that reports no operands; resolution must + # resolve its header word. Regresses JuliaGPU/oneAPI.jl's "#55: invalid integers created + # by alloc_opt", where `SVector(0f0, 0f0)` boxed a zero payload. + JuliaContext() do ctx + mod = parse(LLVM.Module, + "@zero_box = private global { i64, [8 x i8] } zeroinitializer") + gv = globals(mod)["zero_box"] + @test initializer(gv) isa LLVM.ConstantAggregateZero # the folded shape + relocs = GPUCompiler.Relocations(Dict( + GPUCompiler.RelocationSite("zero_box", 0) => GPUCompiler.JuliaValueRef(Float64))) + GPUCompiler.bake_relocations!(mod, relocs) + init = initializer(gv) + @test !(init isa LLVM.ConstantAggregateZero) # rebuilt into explicit fields + header = convert(UInt, LLVM.Constant[operands(init)...][1]) + @test header == GPUCompiler.resolve_relocation_target(GPUCompiler.JuliaValueRef(Float64)) + @test isconstant(gv) + @test isempty(relocs.sites) + end +end + +@testset "cglobal relocation" begin # JIT-private symbols like `jl_get_pgcstack_resolved` (JuliaLang/julia#61527) cannot # be looked up using `jl_cglobal`, so we should only resolve bindings that are # actually loaded from, leaving called functions alone. job, _ = Native.create_job(identity, (Nothing,)) JuliaContext() do ctx + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + word_ptr = ptr("i8") + word_ptr_ptr = ptr(word_ptr) + function_word_ptr(name) = GPUCompiler.supports_typed_pointers(ctx) ? + "i64* bitcast (i64 ()* @$name to i64*)" : "ptr @$name" + mod = parse(LLVM.Module, """ declare void @jl_get_pgcstack_resolved() @@ -746,6 +1005,191 @@ end }""") GPUCompiler.prepare_execution!(job, mod) @test haskey(functions(mod), "jl_get_pgcstack_resolved") + + mod = parse(LLVM.Module, """ + @jl_float32_type = external global $word_ptr + + define $word_ptr @entry() { + %value = load $word_ptr, $word_ptr_ptr @jl_float32_type + ret $word_ptr %value + }""") + GPUCompiler.prepare_execution!(job, mod) + @test !occursin("load $word_ptr, $word_ptr_ptr @jl_float32_type", string(mod)) + @test occursin(r"private .*constant", string(mod)) + + mod = parse(LLVM.Module, """ + @jl_float32_type = external global $word_ptr + + define $word_ptr @entry() { + %value = load $word_ptr, $word_ptr_ptr @jl_float32_type + ret $word_ptr %value + }""") + relocs = GPUCompiler.Relocations() + @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) + site = only(keys(relocs.sites)) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test site.offset == 0 + @test occursin("@$(site.name) = external global i64", string(mod)) + GPUCompiler.emit_patchable_relocations!(mod, relocs) + @test occursin("externally_initialized global i64 0", string(mod)) + + mod = parse(LLVM.Module, """ + declare i64 @jl_float32_type() + + define i64 @entry() { + %value = load i64, $(function_word_ptr("jl_float32_type")) + ret i64 %value + }""") + relocs = GPUCompiler.Relocations() + @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) + site = only(keys(relocs.sites)) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test occursin("@$(site.name) = external global i64", string(mod)) + GPUCompiler.emit_imported_relocations!(mod, relocs) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test isdeclaration(globals(mod)[site.name]) + @test occursin("@$(site.name) = external global i64", string(mod)) + + mod = parse(LLVM.Module, """ + @jl_float32_type = external global $word_ptr + + define $word_ptr @entry() { + %value = load $word_ptr, $word_ptr_ptr @jl_float32_type + ret $word_ptr %value + }""") + relocs = GPUCompiler.Relocations() + @test GPUCompiler.collect_cglobal_relocations!(mod, relocs) + site = only(keys(relocs.sites)) + GPUCompiler.emit_imported_relocations!(mod, relocs) + @test relocs.sites[site] == GPUCompiler.CGlobalRef(:jl_float32_type) + @test isdeclaration(globals(mod)[site.name]) + @test occursin("@$(site.name) = external global i64", string(mod)) + end +end + +@testset "relocation linking" begin + JuliaContext() do ctx + ptr(T) = GPUCompiler.supports_typed_pointers(ctx) ? "$T*" : "ptr" + + function slot_module(name, entry) + parse(LLVM.Module, """ + @$name = external global i64 + + define i64 @$entry() { + %value = load i64, $(ptr("i64")) @$name + ret i64 %value + }""") + end + + site(name, offset=0) = GPUCompiler.RelocationSite(name, offset) + slot_relocs(name, value) = GPUCompiler.Relocations( + Dict(site(name) => GPUCompiler.JuliaValueRef(value))) + + # Equal Julia identities deliberately share a single slot. + dest = slot_module("slot", "first") + dest_relocs = slot_relocs("slot", :shared) + src = slot_module("slot", "second") + src_relocs = slot_relocs("slot", :shared) + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs) + @test collect(keys(dest_relocs.sites)) == [site("slot")] + @test occursin("@slot = external global i64", string(dest)) + + # The name is the slot identity, so conflicting metadata is an error. + dest = slot_module("slot", "first") + dest_relocs = slot_relocs("slot", :first) + src = slot_module("slot", "second") + src_relocs = slot_relocs("slot", :second) + @test_throws ErrorException GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) + + # `only_needed` must keep metadata for imported slots and discard metadata for + # source globals that the LLVM linker did not import. + dest = parse(LLVM.Module, """ + declare i64 @source() + + define i64 @entry() { + %value = call i64 @source() + ret i64 %value + }""") + src = parse(LLVM.Module, """ + @used = external global i64 + @unused = external global i64 + + define i64 @source() { + %value = load i64, $(ptr("i64")) @used + ret i64 %value + }""") + src_relocs = GPUCompiler.Relocations(Dict( + site("used") => GPUCompiler.JuliaValueRef(:used), + site("unused") => GPUCompiler.JuliaValueRef(:unused), + )) + dest_relocs = GPUCompiler.Relocations() + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; + only_needed=true) + @test collect(keys(dest_relocs.sites)) == [site("used")] + @test only(values(dest_relocs.sites)).value === :used + + function interior_module(name, entry) + parse(LLVM.Module, """ + @$name = externally_initialized global { i64, i64 } { i64 0, i64 1 } + + define i64 @$entry() { + %value = load i64, $(ptr("i64")) getelementptr ({ i64, i64 }, $(ptr("{ i64, i64 }")) @$name, i32 0, i32 1) + ret i64 %value + }""") + end + interior_relocs(name, value) = GPUCompiler.Relocations( + Dict(site(name) => GPUCompiler.JuliaValueRef(value))) + + # Identical interior relocation identities merge; conflicting metadata does not. + dest = interior_module("patch", "first_patch") + dest_relocs = interior_relocs("patch", Float64) + src = interior_module("patch", "second_patch") + src_relocs = interior_relocs("patch", Float64) + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs) + @test collect(keys(dest_relocs.sites)) == [site("patch")] + + dest = interior_module("patch", "first_patch") + dest_relocs = interior_relocs("patch", Float64) + src = interior_module("patch", "second_patch") + src_relocs = interior_relocs("patch", Int64) + @test_throws ErrorException GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) + + # Metadata for interior globals not imported under `only_needed` is discarded. + dest = parse(LLVM.Module, """ + declare i64 @source_patch() + define i64 @entry_patch() { + %value = call i64 @source_patch() + ret i64 %value + }""") + src = interior_module("used_patch", "source_patch") + unused = GlobalVariable(src, LLVM.StructType([LLVM.Int64Type(), LLVM.Int64Type()]), + "unused_patch") + initializer!(unused, ConstantStruct(LLVM.Constant[ConstantInt(0), ConstantInt(1)])) + src_relocs = GPUCompiler.Relocations(Dict( + site("used_patch") => GPUCompiler.JuliaValueRef(Float64), + site("unused_patch") => GPUCompiler.JuliaValueRef(Int64))) + dest_relocs = GPUCompiler.Relocations() + GPUCompiler.link_relocatable!(dest, dest_relocs, src, src_relocs; + only_needed=true) + @test collect(keys(dest_relocs.sites)) == [site("used_patch")] + + # A shared site whose source global is missing is an inconsistency, as is a source + # definition with no matching destination global to link against. + dest = slot_module("slot", "first") + dest_relocs = slot_relocs("slot", :shared) + src = LLVM.Module("empty") + src_relocs = slot_relocs("slot", :shared) + @test_throws "Missing source relocation global" GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) + + dest = LLVM.Module("empty") + dest_relocs = interior_relocs("patch", Float64) + src = interior_module("patch", "second_patch") + src_relocs = interior_relocs("patch", Float64) + @test_throws "Missing destination relocation global" GPUCompiler.link_relocatable!( + dest, dest_relocs, src, src_relocs) end end @@ -1100,6 +1544,31 @@ end @test occursin("call void @julia_kernel", ir) end +@testset "Mock Enzyme deferred relocations" begin + # A deferred child that references a Julia value produces its own relocations; those + # must merge into the parent's metadata when the child module is linked in. + mod = @eval module $(gensym()) + import ..Enzyme + child(sym::Symbol) = sym === :deferred_reloc ? 1 : 2 + function parent(sym::Symbol) + ptr = Enzyme.deferred_codegen(typeof(child), Tuple{Symbol}) + return ccall(ptr, Int, (Symbol,), sym) + end + end + + # Keep the merged relocation symbolic so we can inspect it. + job, _ = Native.create_job(mod.parent, (Symbol,); jit=true, validate=false) + JuliaContext() do ctx + ir, meta = GPUCompiler.compile(:llvm, job) + @test !occursin("deferred_codegen", string(ir)) + if GPUCompiler.supports_relocatable_ir() + @test any(values(meta.relocations.sites)) do ref + ref isa GPUCompiler.JuliaValueRef && ref.value === :deferred_reloc + end + end + end +end + @testset "stack allocation intrinsic" begin mod = @eval module $(gensym()) import ..GPUCompiler diff --git a/test/native/precompile.jl b/test/native/precompile.jl index eb9abc4f..282f4f6e 100644 --- a/test/native/precompile.jl +++ b/test/native/precompile.jl @@ -37,22 +37,21 @@ precompile_test_harness("Inference caching") do load_path Results() = new(nothing) end - portable_kernel(x) = x + 1 + persistent_kernel(x) = x + 1 session_kernel(x) = x + 2 - # Attach representative back-end artifacts while the package image is built. The - # portable entry should survive serialization; the session-dependent one should be - # removed by GPUCompiler's pre-output atexit hook. + # A back-end implementing relocation keeps results across sessions. let - job, _ = NativeCompiler.Native.create_job(portable_kernel, (Int,)) + job, _ = NativeCompiler.Native.create_job(persistent_kernel, (Int,); jit=true) precompile(job) - NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "portable" + NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "persistent" end + + # Default back-ends use the session-local store. let job, _ = NativeCompiler.Native.create_job(session_kernel, (Int,)) precompile(job) NativeCompiler.GPUCompiler.cached_results(Results, job).artifact = "session" - NativeCompiler.GPUCompiler.mark_session_dependent!(job) end let @@ -111,10 +110,15 @@ precompile_test_harness("Inference caching") do load_path using NativeBackend - portable_job, _ = NativeCompiler.Native.create_job(NativeBackend.portable_kernel, (Int,)) - portable_res = GPUCompiler.cached_results(NativeBackend.Results, portable_job) - @test portable_res !== nothing - @test portable_res.artifact == "portable" + persistent_job, _ = NativeCompiler.Native.create_job( + NativeBackend.persistent_kernel, (Int,); jit=true) + persistent_res = GPUCompiler.cached_results(NativeBackend.Results, persistent_job) + @test persistent_res !== nothing + if GPUCompiler.supports_relocatable_ir() + @test persistent_res.artifact == "persistent" + else + @test persistent_res.artifact === nothing + end session_job, _ = NativeCompiler.Native.create_job(NativeBackend.session_kernel, (Int,)) session_res = GPUCompiler.cached_results(NativeBackend.Results, session_job) diff --git a/test/ptx.jl b/test/ptx.jl index e98f55d8..661d0f89 100644 --- a/test/ptx.jl +++ b/test/ptx.jl @@ -29,19 +29,34 @@ end end @testset "global variable relocation" begin - # references to Julia objects (`julia.constgv` globals, e.g. Symbol literals) must - # survive until `relocate_gvs!` bakes in their addresses at the toplevel link step. - # they used to be kept alive as internal globals with a null initializer, which the - # GlobalOpt run in `finish_module!` folded away, constant-folding any comparison - # against them (JuliaGPU/CUDA.jl#3185: kernels specialized on Symbols misbehaved). + # Julia-object references must remain declarations until relocation lowering. Null + # definitions would let GlobalOpt fold comparisons against them. mod = @eval module $(gensym()) kernel(name::Symbol) = name === :var ? 1 : 2 end + # `patch=true` keeps the slot symbolic instead of resolving it into the IR. ir = sprint() do io - PTX.code_llvm(io, mod.kernel, Tuple{Symbol}; dump_module=true) + PTX.code_llvm(io, mod.kernel, Tuple{Symbol}; dump_module=true, patch=true) end - addr = UInt64(pointer_from_objref(:var)) - @test occursin(string(addr), ir) + # Julia embeds the Symbol pointer before exposing the IR to GPUCompiler when it + # can't emit relocatable global metadata; otherwise it stays a symbolic slot we preserve. + if GPUCompiler.supports_relocatable_ir() + @test occursin("@jl_sym_var_", ir) + else + @test occursin("@jl_sym_var_", ir) || occursin("inttoptr", ir) + end + @test !occursin("@\"jl_sym#", ir) +end + +@testset "Julia value global names" begin + # Julia's external codegen can name the declaration for this Symbol with `#`. + # The final PTX path must see the sanitized Julia value global, not the original + # declaration name rejected by NVPTX. + mod = @eval module $(gensym()) + const unusual_symbol = Symbol("value#global") + kernel(name::Symbol) = (name === unusual_symbol; return) + end + @test PTX.code_execution(mod.kernel, Tuple{Symbol}) !== nothing end @testset "boxed Bool singleton relocation" begin @@ -72,6 +87,28 @@ end end end +@testset "boxed header relocation" begin + mod = @eval module $(gensym()) + @noinline produce(cond::Bool, value::Int32) = cond ? value : 1.5 + function consume(cond::Bool, value::Int32) + x = produce(cond, value) + x isa Float64 && return x + return 0.0 + end + end + # `patch=true` keeps the interior header relocation symbolic (externally_initialized). + ir = sprint(io->PTX.code_llvm(io, mod.consume, Tuple{Bool,Int32}; + dump_module=true, patch=true)) + # As with Symbol literals above, Julia embeds the type pointer when it can't emit + # relocatable global metadata; otherwise GPUCompiler represents it as a relocation. + if GPUCompiler.supports_relocatable_ir() + @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) + else + @test occursin(r"@[A-Za-z0-9_]+_box = externally_initialized global", ir) || + occursin("inttoptr", ir) + end +end + @testset "kernel functions" begin @testset "kernel argument attributes" begin mod = @eval module $(gensym()) @@ -200,6 +237,24 @@ if :NVPTX in LLVM.backends() end end +@testset "patchable relocation" begin + # A patching back-end (like CUDA.jl) keeps relocations symbolic; the patchable box must + # survive lowering into the generated PTX as a `.global` for the loader to write. + if GPUCompiler.supports_relocatable_ir() + mod = @eval module $(gensym()) + @noinline produce(cond::Bool, value::Int32) = cond ? value : 1.5 + function consume(cond::Bool, value::Int32) + x = produce(cond, value) + x isa Float64 && return x + return 0.0 + end + end + ptx = sprint(io->PTX.code_native(io, mod.consume, Tuple{Bool,Int32}; + dump_module=true, patch=true)) + @test occursin(r"\.global .*_box", ptx) + end +end + @testset "child functions" begin # we often test using @noinline child functions, so test whether these survive # (despite not having side-effects)