JIT session error triggered by precompilation

[pxl-th]

Using PrecompileTools.jl to precompile GaussianSpaltting.jl triggers following error on 1.10 when executing code after precompilation:

JIT session error: Symbols not found: [ julia.gpu.state_getter ]
JIT session error: Failed to materialize symbols: { (JuliaOJIT, { julia_#throw_inexacterror_2603 }) }
JIT session error: Failed to materialize symbols: { (JuliaOJIT, { julia_#GaussianRasterizer#16_2598 }) }

MWE:

The error seems to be triggered at the moment of calling GaussianRasterizer ctor.

using CUDA
using cuDNN
using GaussianSplatting

function main()
    kab = GaussianSplatting.gpu_backend()

    points = CuArray(rand(Float32, 3, 128))
    colors = CuArray(rand(Float32, 3, 128))
    scales = CuArray(rand(Float32, 3, 128))

    camera = GaussianSplatting.Camera(; fx=100f0, fy=100f0, width=256, height=256)
    gaussians = GaussianSplatting.GaussianModel(points, colors, scales; max_sh_degree=0)
    rasterizer = GaussianSplatting.GaussianRasterizer(kab, camera; auxiliary=false)
    return
end
main()

[maleadt]

cc @vchuravy

[pxl-th]

With AMDGPU on 1.10 there is no such issue, howerver, 1.11 fails during precompilation.
Without precompilation the package works fine.

julia> using AMDGPU, GaussianSplatting
Precompiling GaussianSplattingAMDGPUExt...
Info Given GaussianSplattingAMDGPUExt was explicitly requested, output will be shown live 
[ Info: Precompiling for `AMDGPU.ROCKernels.ROCBackend()` GPU backend.
[ Info: Done precompiling!
LLVM ERROR: Cannot select: intrinsic %llvm.amdgcn.dispatch.ptr

[19065] signal 6 (-6): Aborted
in expression starting at none:0
Allocations: 128924616 (Pool: 128919573; Big: 5043); GC: 41
┌ Error: Error during loading of extension GaussianSplattingAMDGPUExt of GaussianSplatting, use `Base.retry_load_extensions()` to retry.
│   exception =
│    1-element ExceptionStack:
│    Failed to precompile GaussianSplattingAMDGPUExt [ce7b01fd-24c6-5d8a-8407-52c975a293bc] to "/home/pxlth/.julia/compiled/v1.11/GaussianSplattingAMDGPUExt/jl_1oLeu5".
│    Stacktrace:
│      [1] error(s::String)
│        @ Base ./error.jl:35
│      [2] compilecache(pkg::Base.PkgId, path::String, internal_stderr::IO, internal_stdout::IO, keep_loaded_modules::Bool; flags::Cmd, cacheflags::Base.CacheFlags, reasons::Dict{String, Int64})
│        @ Base ./loading.jl:3032
│      [3] (::Base.var"#1080#1081"{Base.PkgId})()
│        @ Base ./loading.jl:2420
│      [4] mkpidlock(f::Base.var"#1080#1081"{Base.PkgId}, at::String, pid::Int32; kwopts::@Kwargs{stale_age::Int64, wait::Bool})
│        @ FileWatching.Pidfile ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/FileWatching/src/pidfile.jl:95
│      [5] #mkpidlock#6
│        @ ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/FileWatching/src/pidfile.jl:90 [inlined]
│      [6] trymkpidlock(::Function, ::Vararg{Any}; kwargs::@Kwargs{stale_age::Int64})
│        @ FileWatching.Pidfile ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/FileWatching/src/pidfile.jl:116
│      [7] #invokelatest#2
│        @ ./essentials.jl:1045 [inlined]
│      [8] invokelatest
│        @ ./essentials.jl:1040 [inlined]
│      [9] maybe_cachefile_lock(f::Base.var"#1080#1081"{Base.PkgId}, pkg::Base.PkgId, srcpath::String; stale_age::Int64)
│        @ Base ./loading.jl:3550
│     [10] maybe_cachefile_lock
│        @ ./loading.jl:3547 [inlined]
│     [11] _require(pkg::Base.PkgId, env::Nothing)
│        @ Base ./loading.jl:2416
│     [12] __require_prelocked(uuidkey::Base.PkgId, env::Nothing)
│        @ Base ./loading.jl:2243
│     [13] #invoke_in_world#3
│        @ ./essentials.jl:1077 [inlined]
│     [14] invoke_in_world
│        @ ./essentials.jl:1074 [inlined]
│     [15] _require_prelocked
│        @ ./loading.jl:2230 [inlined]
│     [16] _require_prelocked
│        @ ./loading.jl:2229 [inlined]
│     [17] run_extension_callbacks(extid::Base.ExtensionId)
│        @ Base ./loading.jl:1430
│     [18] run_extension_callbacks(pkgid::Base.PkgId)
│        @ Base ./loading.jl:1465
│     [19] run_package_callbacks(modkey::Base.PkgId)
│        @ Base ./loading.jl:1293
│     [20] __require_prelocked(uuidkey::Base.PkgId, env::String)
│        @ Base ./loading.jl:2254
│     [21] #invoke_in_world#3
│        @ ./essentials.jl:1077 [inlined]
│     [22] invoke_in_world
│        @ ./essentials.jl:1074 [inlined]
│     [23] _require_prelocked(uuidkey::Base.PkgId, env::String)
│        @ Base ./loading.jl:2230
│     [24] macro expansion
│        @ ./loading.jl:2169 [inlined]
│     [25] macro expansion
│        @ ./lock.jl:273 [inlined]
│     [26] __require(into::Module, mod::Symbol)
│        @ Base ./loading.jl:2126
│     [27] #invoke_in_world#3
│        @ ./essentials.jl:1077 [inlined]
│     [28] invoke_in_world
│        @ ./essentials.jl:1074 [inlined]
│     [29] require(into::Module, mod::Symbol)
│        @ Base ./loading.jl:2119
│     [30] eval
│        @ ./boot.jl:429 [inlined]
│     [31] eval_user_input(ast::Any, backend::REPL.REPLBackend, mod::Module)
│        @ REPL ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/REPL/src/REPL.jl:224
│     [32] repl_backend_loop(backend::REPL.REPLBackend, get_module::Function)
│        @ REPL ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/REPL/src/REPL.jl:321
│     [33] start_repl_backend(backend::REPL.REPLBackend, consumer::Any; get_module::Function)
│        @ REPL ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/REPL/src/REPL.jl:306
│     [34] run_repl(repl::REPL.AbstractREPL, consumer::Any; backend_on_current_task::Bool, backend::Any)
│        @ REPL ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/REPL/src/REPL.jl:462
│     [35] run_repl(repl::REPL.AbstractREPL, consumer::Any)
│        @ REPL ~/bin/julia-1.11.0-rc2/share/julia/stdlib/v1.11/REPL/src/REPL.jl:448
│     [36] (::Base.var"#1137#1139"{Bool, Symbol, Bool})(REPL::Module)
│        @ Base ./client.jl:441
│     [37] #invokelatest#2
│        @ ./essentials.jl:1043 [inlined]
│     [38] invokelatest
│        @ ./essentials.jl:1040 [inlined]
│     [39] run_main_repl(interactive::Bool, quiet::Bool, banner::Symbol, history_file::Bool, color_set::Bool)
│        @ Base ./client.jl:425
│     [40] repl_main
│        @ ./client.jl:562 [inlined]
│     [41] _start()
│        @ Base ./client.jl:536
└ @ Base loading.jl:1436

[KSepetanc]

PackageCompiler fails to compile CUDA with compile=all. As suggested in Discourse, this issue might be related or the root of it.

[KSepetanc]

I wish to raise the importance of this issue as it blocks compilation of any CUDA.jl related project (using both PrecompileTools and PackageCompiler).

[maleadt]

MWE:

module MWE

using GPUCompiler, PrecompileTools

struct CompilerParams <: AbstractCompilerParams end

@inline threadfence_block() = ccall("llvm.nvvm.membar.cta", llvmcall, Cvoid, ())
kernel() = threadfence_block()

@setup_workload begin
    source = methodinstance(typeof(kernel), Tuple{})
    target = PTXCompilerTarget(cap=v"5.0")
    params = CompilerParams()
    config = CompilerConfig(target, params)
    job = CompilerJob(source, config)

    @compile_workload begin
        JuliaContext() do ctx
            GPUCompiler.compile(:llvm, job)
        end
    end
end

end # module MWE

Precompiling MWE...
LLVM ERROR: Cannot select: intrinsic %llvm.nvvm.membar.cta

[KSepetanc]

It is either random or it got worse with Julia 1.12. I am now getting this error with default settings when compiling with PackageCompiler. Previously I got error only with --compile=all.

Should we open issue on Julia github as probably solution should be seeked there?

[maleadt]

It would probably help to reduce this to an AbstractInterpreter example, i.e. not using GPUCompiler, before filing upstream.

[KSepetanc]

The error occurs with JuliaC too (no trim):

LLVM ERROR: Cannot select: intrinsic %llvm.nvvm.barrier0

[21976] signal 22: SIGABRT
in expression starting at none:0

[KSepetanc]

I know it is not standard to use AI for these things, but I needed to since I am not an expert in this. I used Gemini to try to make MWE. What it did is create counter example without GPUCompiler that actually needs to crash, but it does not, meaning compiler silently skips instructions potentially causing much more serious issue than leaked GPU instructions, i.e. silent no operation. I was skeptical about this so I crosschecked by asking ChatGPT without code remarks to say what should happen with code and it said it should crash (but it does not even when using -O0 setting). Feedback is much appreciated on this.

module IntrinsicLeakMWE

using InteractiveUtils

# 1. Define the NVPTX-specific intrinsic
@inline function threadfence_block()
    Base.llvmcall(
        """
        ; This intrinsic should only exist on the NVPTX (GPU) backend.
        call void @llvm.nvvm.membar.cta()
        ret void
        declare void @llvm.nvvm.membar.cta()
        """,
        Cvoid, Tuple{}
    )
end

function kernel()
    threadfence_block()
    return nothing
end

# 2. Crash Trigger (Execute immediately upon include)

println("\n--- Attempting to force host compilation of GPU intrinsic ---")

try
    # This line forces the full compiler pipeline, where the host LLVM backend fails
    # when it encounters the GPU-only instruction.
    InteractiveUtils.code_native(devnull, kernel, Tuple{})
    
    # If the code reaches here, the bug is the silent discard
    println("🚨 WARNING: The compiler successfully ignored the intrinsic instead of crashing.")
    
catch e
    if e isa Base.Exception && contains(sprint(showerror, e), "llvm.nvvm.membar.cta")
        # This is the desired crash and the bug proof
        println("\n✅ CRASH HIT: LLVM ERROR: Cannot select: intrinsic %llvm.nvvm.membar.cta")
    else
        println("\n❌ UNEXPECTED ERROR during compilation.")
        showerror(stdout, e)
    end
    rethrow(e)
end

end # module IntrinsicLeakMWE

[KSepetanc]

@maleadt I tried your MWE, but it fails only on 1.11 (it runs ok with 1.10 and 1.12).
As side note, I added GPUCompiler.runtime_module(::GPUCompiler.CompilerJob{<:Any, CompilerParams}) = MWE
after struct definition. It would not work without it on 1.12 (ERROR: LoadError: Not implemented).

[KSepetanc]

I was unable to recreate failure running PrecompileTools and Julia 1.12 even when using full scale test case with CUDA (on 1.11 it fails).

However, PackageCompiler and JuliaC still fail with 1.12.
The first with LLVM ERROR: Broken module found error and function being call with register value rather than compile time constant. The second errors with LLVM ERROR: Cannot select: intrinsic %llvm.nvvm.barrier0 as stated above.

This led me to try again with different approach. We already know creating system image with CUDA.jl fails (Cannot select error) so I tried to narrow it down. Including the GPUCompiler in the system image it self already fails (create_sysimage(["GPUCompiler"]; sysimage_path="ExampleSysimage.so")). The error from below led me to check src/runtime.jl, in particular function compile (lines 69-90). There is FIXME sign that appears to be related to this. Including CUDA to system image fails as already including GPUCompiler fails. Could it be that this is the remaining culprit (at least on v1.12)?

    # FIXME: if the function is a symbol, implying it will be specified by the target,
    #        we won't be able to call this function here or we'll get UndefVarErrors.
    #        work around that by generating an llvmcall stub. can we do better by
    #        using the new nonrecursive codegen to handle function lookup ourselves?

C:/Users/karlo/.julia/artifacts/b17bda08a19173572926f43a48aad5ef3d845e7c/extracted_files/mingw64/bin/../lib/gcc/x86_64-w64-mingw32/14.2.0/../../../../x86_64-w64-mingw32/bin/ld.exe: C:\Users\karlo\AppData\Local\Temp\jl_4VIdiJ41cl-o.a(text#0.o): in function `julia_signal_exception_106257':
C:\Users\karlo\.julia\packages\GPUCompiler\j4HFa\src/runtime.jl:85:(.text+0x7c3d6): undefined reference to `gpu_signal_exception'
C:/Users/karlo/.julia/artifacts/b17bda08a19173572926f43a48aad5ef3d845e7c/extracted_files/mingw64/bin/../lib/gcc/x86_64-w64-mingw32/14.2.0/../../../../x86_64-w64-mingw32/bin/ld.exe: C:\Users\karlo\AppData\Local\Temp\jl_4VIdiJ41cl-o.a(text#7.o): in function `malloc':
C:\Users\karlo\.julia\packages\GPUCompiler\j4HFa\src/runtime.jl:85:(.text+0x147a6a): undefined reference to `gpu_malloc'
C:/Users/karlo/.julia/artifacts/b17bda08a19173572926f43a48aad5ef3d845e7c/extracted_files/mingw64/bin/../lib/gcc/x86_64-w64-mingw32/14.2.0/../../../../x86_64-w64-mingw32/bin/ld.exe: C:\Users\karlo\AppData\Local\Temp\jl_4VIdiJ41cl-o.a(text#7.o): in function `report_oom':
C:\Users\karlo\.julia\packages\GPUCompiler\j4HFa\src/runtime.jl:85:(.text+0x147a7e): undefined reference to `gpu_report_oom'
collect2.exe: error: ld returned 1 exit status

[apozharski]

I know it is not standard to use AI for these things, but I needed to since I am not an expert in this. I used Gemini to try to make MWE. What it did is create counter example without GPUCompiler that actually needs to crash, but it does not, meaning compiler silently skips instructions potentially causing much more serious issue than leaked GPU instructions, i.e. silent no operation. I was skeptical about this so I crosschecked by asking ChatGPT without code remarks to say what should happen with code and it said it should crash (but it does not even when using -O0 setting). Feedback is much appreciated on this.

module IntrinsicLeakMWE

using InteractiveUtils

# 1. Define the NVPTX-specific intrinsic
@inline function threadfence_block()
    Base.llvmcall(
        """
        ; This intrinsic should only exist on the NVPTX (GPU) backend.
        call void @llvm.nvvm.membar.cta()
        ret void
        declare void @llvm.nvvm.membar.cta()
        """,
        Cvoid, Tuple{}
    )
end

function kernel()
    threadfence_block()
    return nothing
end

# 2. Crash Trigger (Execute immediately upon include)

println("\n--- Attempting to force host compilation of GPU intrinsic ---")

try
    # This line forces the full compiler pipeline, where the host LLVM backend fails
    # when it encounters the GPU-only instruction.
    InteractiveUtils.code_native(devnull, kernel, Tuple{})
    
    # If the code reaches here, the bug is the silent discard
    println("🚨 WARNING: The compiler successfully ignored the intrinsic instead of crashing.")
    
catch e
    if e isa Base.Exception && contains(sprint(showerror, e), "llvm.nvvm.membar.cta")
        # This is the desired crash and the bug proof
        println("\n✅ CRASH HIT: LLVM ERROR: Cannot select: intrinsic %llvm.nvvm.membar.cta")
    else
        println("\n❌ UNEXPECTED ERROR during compilation.")
        showerror(stdout, e)
    end
    rethrow(e)
end

end # module IntrinsicLeakMWE

I will spare you the diatribe about how the practice of adding stochastic noise to the process of open source development is unhelpful and wastes peoples time, but forgive me my frustrated tone.

As per usual, the stochastic parrots masquerading as intelligence you are using are being confidently incorrect. Had you checked and not redirected the code_native output to devnull you would have seen that the llvm code is not being compiled because it is trying to declare the intrinsic as a new function for some reason:

--- Attempting to force host compilation of GPU intrinsic ---
        .text
        .file   "kernel"
        .section        .ltext,"axl",@progbits
        .globl  julia_kernel_2800               # -- Begin function julia_kernel_2800
        .p2align        4, 0x90
        .type   julia_kernel_2800,@function
julia_kernel_2800:                      # @julia_kernel_2800
; Function Signature: kernel()
; ┌ @ /home/anton/test/libmad/ihateai.jl:18 within `kernel`
# %bb.0:                                # %top
        push    rbp
; │ @ /home/anton/test/libmad/ihateai.jl:19 within `kernel`
; │┌ @ /home/anton/test/libmad/ihateai.jl:7 within `threadfence_block`
        movabs  rdi, offset ".L_j_str_Failed to parse LLVM asse...#1"
        movabs  rax, offset ijl_error
        mov     rbp, rsp
        call    rax
.Lfunc_end0:
        .size   julia_kernel_2800, .Lfunc_end0-julia_kernel_2800
; └└
                                        # -- End function
        .type   ".L_j_str_Failed to parse LLVM asse...#1",@object # @"_j_str_Failed to parse LLVM asse...#1"
        .section        .rodata.str1.1,"aMSl",@progbits,1
".L_j_str_Failed to parse LLVM asse...#1":
        .asciz  "Failed to parse LLVM assembly: \n<string>:5:1: error: expected instruction opcode\ndeclare void @llvm.nvvm.membar.cta()\n^\n"
        .size   ".L_j_str_Failed to parse LLVM asse...#1", 121

        .section        ".note.GNU-stack","",@progbits
🚨 WARNING: The compiler successfully ignored the intrinsic instead of crashing.
Main.IntrinsicLeakMWE

This is generally illegal to do in a define scope, which the string passed to Base.llvmcall is wrapped in (see the julia documentation, and the llvm documentation).

If you remove declare void @llvm.nvvm.membar.cta() then you get hard crash as expected:

--- Attempting to force host compilation of GPU intrinsic ---
LLVM ERROR: Cannot select: intrinsic %llvm.nvvm.membar.cta

[263999] signal 6 (-6): Aborted

In other words, nothing is being skipped, the bug we are trying to find is still at large, and plausible string generators reveal themselves to be nothing but that, unthinking plausible string generators.