Make the runtime library disk cache best-effort. (#869)

The cache directory is shared across processes and can be deleted at any
point (e.g. by `reset_runtime()`), racing concurrent compilation.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: maleadt

src/rtlib.jl

@@ -139,23 +139,35 @@ end
     name = "runtime_$(slug).bc"
     path = joinpath(compile_cache, name)
 
-    if !ispath(path)
-        @debug "Building the GPU runtime library at $path"
-        mkpath(compile_cache)
-        lib = build_runtime(job)
+    # the cache is shared across processes and may disappear at any point
+    # (e.g. `reset_runtime()` in another process), so treat it as best-effort
+    if ispath(path)
+        try
+            return parse(LLVM.Module, MemoryBufferFile(path); lazy=true)
+        catch err
+            @debug "Failed to load cached GPU runtime library; rebuilding" exception=(err, catch_backtrace())
+        end
+    end
 
+    @debug "Building the GPU runtime library at $path"
+    lib = build_runtime(job)
+
+    try
         # atomic write to disk
-        temp_path, io = mktemp(dirname(path); cleanup=false)
+        mkpath(compile_cache)
+        temp_path, io = mktemp(compile_cache; cleanup=false)
         write(io, lib)
         close(io)
         @static if VERSION >= v"1.12.0-DEV.1023"
             mv(temp_path, path; force=true)
         else
             Base.rename(temp_path, path, force=true)
         end
+    catch err
+        @warn "Failed to cache GPU runtime library" exception=(err, catch_backtrace()) maxlog=1
     end
 
-    return parse(LLVM.Module, MemoryBufferFile(path); lazy=true)
+    return lib
 end
 
 # remove the existing cache

test/ptx.jl

@@ -32,6 +32,34 @@ end
     @test occursin("[1 x i64] %state", ir)
 end
 
+@testset "runtime cache disappearing" begin
+    # the cache may be deleted or unusable at any point; compilation should keep working
+    mod = @eval module $(gensym())
+        kernel() = return
+    end
+    old_cache = GPUCompiler.compile_cache
+    try
+        GPUCompiler.compile_cache = mktempdir()
+        PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true)
+        @test !isempty(readdir(GPUCompiler.compile_cache))
+
+        # remove the cache directory altogether
+        GPUCompiler.reset_runtime()
+        PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true)
+        @test !isempty(readdir(GPUCompiler.compile_cache))
+
+        # make the cache unwritable; compilation should proceed with only a warning
+        GPUCompiler.reset_runtime()
+        write(GPUCompiler.compile_cache, "not a directory")
+        @test_logs (:warn, r"Failed to cache GPU runtime library") match_mode=:any begin
+            PTX.code_llvm(devnull, mod.kernel, Tuple{}; kernel=true)
+        end
+        rm(GPUCompiler.compile_cache)
+    finally
+        GPUCompiler.compile_cache = old_cache
+    end
+end
+
 @testset "kernel functions" begin
 @testset "kernel argument attributes" begin
     mod = @eval module $(gensym())