JuliaGPU / JuliaGPU/KernelAbstractions.jl
Use with nested StructArrays
- Dominant language
- Julia
- Stars
- 523
- Forks
- 88
- Avg merge
- 1d 11h
- Merged PRs (30d)
- 25
Description
We're trying to combine KernelAbstractions with StructArrays, but get problems when nested `StructArray`s are used. A reproducible example:
```julia
using CUDA, KernelAbstractions, StructArrays
@kernel function copy_kernel!(A, @Const(B))
I = @index(Global)
@inbounds A[I] = B[I]
end
function mycopy!(A, B)
@assert size(A) == size(B)
copy_kernel!(CUDADevice(), 256)(A, B, ndrange=length(A))
end
struct Foo
x::Float64
y::Tuple{Float64}
end
A = CuArray(ones(10))
B = similar(A)
p = StructArray{Tuple{Float64}}((B,))
C = StructArray{Foo}((A,p))
D = similar(C)
wait(mycopy!(D,C))
```
gives:
```
(structarrays) pkg> st
Status `/central/home/spjbyrne/misc/structarrays/Project.toml`
[052768ef] CUDA v1.3.3
[63c18a36] KernelAbstractions v0.4.2
[09ab397b] StructArrays v0.4.4
julia> include("repro.jl")
[ Info: Precompiling StructArrays [09ab397b-f2b6-538f-b94a-2f83cf4a842a]
ERROR: LoadError: InvalidIRError: compiling kernel gpu_copy_kernel!(Cassette.Context{nametype(CUDACtx),KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.DynamicCheck,Nothing,CartesianIndices{1,Tuple{Base.OneTo{Int64}}},KernelAbstractions.NDIteration.NDRange{1,KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.StaticSize{(256,)},CartesianIndices{1,Tuple{Base.OneTo{Int64}}},Nothing}},Nothing,KernelAbstractions.var"##PassType#253",Nothing,Cassette.DisableHooks}, typeof(gpu_copy_kernel!), StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64}, StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64}) resulted in invalid LLVM IR
Reason: unsupported dynamic function invocation (call to overdub(overdub_context::Cassette.Context, overdub_arguments...) in Cassette at /home/spjbyrne/.julia/packages/Cassette/158rp/src/overdub.jl:586)
Stacktrace:
[1] setindex! at /home/spjbyrne/.julia/packages/StructArrays/OtfvU/src/structarray.jl:166
[2] #58 at /home/spjbyrne/.julia/packages/StructArrays/OtfvU/src/structarray.jl:166
[3] macro expansion at /home/spjbyrne/.julia/packages/StructArrays/OtfvU/src/utils.jl:46
[4] foreachfield_gen at /home/spjbyrne/.julia/packages/StructArrays/OtfvU/src/utils.jl:46
[5] foreachfield at /home/spjbyrne/.julia/packages/StructArrays/OtfvU/src/utils.jl:51
[6] setindex! at /home/spjbyrne/.julia/packages/StructArrays/OtfvU/src/structarray.jl:166
[7] macro expansion at /central/home/spjbyrne/misc/structarrays/repro.jl:4
[8] gpu_copy_kernel! at /home/spjbyrne/.julia/packages/KernelAbstractions/rFWPd/src/macros.jl:80
[9] overdub at /home/spjbyrne/.julia/packages/Cassette/158rp/src/overdub.jl:0
Stacktrace:
[1] check_ir(::GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget,CUDA.CUDACompilerParams}, ::LLVM.Module) at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/validation.jl:123
[2] macro expansion at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/driver.jl:241 [inlined]
[3] macro expansion at /home/spjbyrne/.julia/packages/TimerOutputs/dVnaw/src/TimerOutput.jl:206 [inlined]
[4] codegen(::Symbol, ::GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget,CUDA.CUDACompilerParams}; libraries::Bool, deferred_codegen::Bool, optimize::Bool, strip::Bool, validate::Bool, only_entry::Bool) at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/driver.jl:239
[5] compile(::Symbol, ::GPUCompiler.CompilerJob{GPUCompiler.PTXCompilerTarget,CUDA.CUDACompilerParams}; libraries::Bool, deferred_codegen::Bool, optimize::Bool, strip::Bool, validate::Bool, only_entry::Bool) at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/driver.jl:39
[6] compile at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/driver.jl:35 [inlined]
[7] _cufunction(::GPUCompiler.FunctionSpec{typeof(Cassette.overdub),Tuple{Cassette.Context{nametype(CUDACtx),KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.DynamicCheck,Nothing,CartesianIndices{1,Tuple{Base.OneTo{Int64}}},KernelAbstractions.NDIteration.NDRange{1,KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.StaticSize{(256,)},CartesianIndices{1,Tuple{Base.OneTo{Int64}}},Nothing}},Nothing,KernelAbstractions.var"##PassType#253",Nothing,Cassette.DisableHooks},typeof(gpu_copy_kernel!),StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64},StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64}}}; kwargs::Base.Iterators.Pairs{Symbol,Int64,Tuple{Symbol},NamedTuple{(:maxthreads,),Tuple{Int64}}}) at /home/spjbyrne/.julia/packages/CUDA/dZvbp/src/compiler/execution.jl:310
[8] check_cache(::typeof(CUDA._cufunction), ::GPUCompiler.FunctionSpec{typeof(Cassette.overdub),Tuple{Cassette.Context{nametype(CUDACtx),KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.DynamicCheck,Nothing,CartesianIndices{1,Tuple{Base.OneTo{Int64}}},KernelAbstractions.NDIteration.NDRange{1,KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.StaticSize{(256,)},CartesianIndices{1,Tuple{Base.OneTo{Int64}}},Nothing}},Nothing,KernelAbstractions.var"##PassType#253",Nothing,Cassette.DisableHooks},typeof(gpu_copy_kernel!),StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64},StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64}}}, ::UInt64; kwargs::Base.Iterators.Pairs{Symbol,Int64,Tuple{Symbol},NamedTuple{(:maxthreads,),Tuple{Int64}}}) at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/cache.jl:24
[9] gpu_copy_kernel! at ./none:0 [inlined]
[10] cached_compilation(::typeof(CUDA._cufunction), ::GPUCompiler.FunctionSpec{typeof(Cassette.overdub),Tuple{Cassette.Context{nametype(CUDACtx),KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.DynamicCheck,Nothing,CartesianIndices{1,Tuple{Base.OneTo{Int64}}},KernelAbstractions.NDIteration.NDRange{1,KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.StaticSize{(256,)},CartesianIndices{1,Tuple{Base.OneTo{Int64}}},Nothing}},Nothing,KernelAbstractions.var"##PassType#253",Nothing,Cassette.DisableHooks},typeof(gpu_copy_kernel!),StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64},StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64}}}, ::UInt64; kwargs::Base.Iterators.Pairs{Symbol,Int64,Tuple{Symbol},NamedTuple{(:maxthreads,),Tuple{Int64}}}) at /home/spjbyrne/.julia/packages/GPUCompiler/GKp4B/src/cache.jl:0
[11] cufunction(::typeof(Cassette.overdub), ::Type{Tuple{Cassette.Context{nametype(CUDACtx),KernelAbstractions.CompilerMetadata{KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.DynamicCheck,Nothing,CartesianIndices{1,Tuple{Base.OneTo{Int64}}},KernelAbstractions.NDIteration.NDRange{1,KernelAbstractions.NDIteration.DynamicSize,KernelAbstractions.NDIteration.StaticSize{(256,)},CartesianIndices{1,Tuple{Base.OneTo{Int64}}},Nothing}},Nothing,KernelAbstractions.var"##PassType#253",Nothing,Cassette.DisableHooks},typeof(gpu_copy_kernel!),StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64},StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global},StructArray{Tuple{Float64},1,Tuple{CuDeviceArray{Float64,1,CUDA.AS.Global}},Int64}}},Int64}}}; name::String, kwargs::Base.Iterators.Pairs{Symbol,Int64,Tuple{Symbol},NamedTuple{(:maxthreads,),Tuple{Int64}}}) at /home/spjbyrne/.julia/packages/CUDA/dZvbp/src/compiler/execution.jl:298
[12] macro expansion at /home/spjbyrne/.julia/packages/CUDA/dZvbp/src/compiler/execution.jl:109 [inlined]
[13] (::KernelAbstractions.Kernel{CUDADevice,KernelAbstractions.NDIteration.StaticSize{(256,)},KernelAbstractions.NDIteration.DynamicSize,typeof(gpu_copy_kernel!)})(::StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuArray{Float64,1},StructArray{Tuple{Float64},1,Tuple{CuArray{Float64,1}},Int64}}},Int64}, ::Vararg{StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuArray{Float64,1},StructArray{Tuple{Float64},1,Tuple{CuArray{Float64,1}},Int64}}},Int64},N} where N; ndrange::Int64, dependencies::Nothing, workgroupsize::Nothing, progress::Function) at /home/spjbyrne/.julia/packages/KernelAbstractions/rFWPd/src/backends/cuda.jl:172
[14] mycopy!(::StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuArray{Float64,1},StructArray{Tuple{Float64},1,Tuple{CuArray{Float64,1}},Int64}}},Int64}, ::StructArray{Foo,1,NamedTuple{(:x, :y),Tuple{CuArray{Float64,1},StructArray{Tuple{Float64},1,Tuple{CuArray{Float64,1}},Int64}}},Int64}) at /central/home/spjbyrne/misc/structarrays/repro.jl:8
[15] top-level scope at /central/home/spjbyrne/misc/structarrays/repro.jl:19
[16] include(::String) at ./client.jl:457
[17] top-level scope at REPL[4]:1
in expression starting at /central/home/spjbyrne/misc/structarrays/repro.jl:19
```
cc: @jb-mackay @dennisYatunin
Contributor guide
No contributing guide indexed for this repository
Research direction
Reproduce the nested StructArray CUDA failure from repro.jl using the versions and kernel shown in the report. Read the StructArrays setindex! and foreachfield frames in the trace, then trace the KernelAbstractions CUDA kernel entry point. Done means the provided copy_kernel! example compiles and copies nested StructArrays without InvalidIRError.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- julia
- Domain
- backend
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 35/100