Error on differentiating through a function allocating CuArray (cuMemcpyHtoDAsync_v2)
- Dominant language
- Julia
- Stars
- 586
- Forks
- 108
- Avg merge
- 1d 5h
- Merged PRs (30d)
- 44
Description
The following example fails at forward or reverse mode since there is apparently no rule for the `cuMemcpyHtoDAsync_v2` CUDA function. Is this expected or is it a missing feature? Thanks!
```julia
using CUDA
using Enzyme
function f(θ)
h = [θ]
d = CuArray(h) # host-device copy (cuMemcpyHtoDAsync_v2)
return sum(d)
end
@info "primal" f(1.0) # 1.0
try
fwd, = Enzyme.autodiff(Forward, f, Duplicated, Duplicated(1.0, 1.0))
@info "Forward mode" fwd
catch e
@error "Forward mode failed" exception=(e, catch_backtrace())
end
try
rev, = Enzyme.autodiff(Reverse, f, Active, Active(1.0))[1]
@info "Reverse mode" rev
catch e
@error "Reverse mode failed" exception=(e, catch_backtrace())
end
```
Error stracktrace
```
┌ Info: primal
└ f(1.0) = 1.0
┌ Error: Forward mode failed
│ exception =
│ EnzymeNoDerivativeError:
│ No forward mode derivative found for cuMemcpyHtoDAsync_v2
│ at context: %10 = call i32 @cuMemcpyHtoDAsync_v2(i64 %1, i64 %2, i64 8, i64 %9) #480, !dbg !686
│
│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424
│
│ Failure within method:
│
│ var"#unsafe_copyto!#1027"(::CuStream, ::Bool, ::typeof(unsafe_copyto!), ::CuPtr{Float64}, ::Ptr{Float64}, ::Int64)
│ @ CUDACore ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421
│
│ Hint: catch this exception as `err` and call `code_typed(err)` to inspect the surrounding code.
│
│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143 [inlined]
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529 [inlined]
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34 [inlined]
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35 [inlined]
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33 [inlined]
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424
│ [7] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421 [inlined]
│ [8] #1215
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:598 [inlined]
│ [9] #context!#1079
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:169 [inlined]
│ [10] context!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:164 [inlined]
│ [11] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:589
│ [12] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:547 [inlined]
│ [13] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:551 [inlined]
│ [14] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:439 [inlined]
│ [15] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:443 [inlined]
│ [16] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:452 [inlined]
│ [17] f
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:24 [inlined]
│ [18] fwddiffejulia_f_347701wrap
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:0
│ [19] macro expansion
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6665 [inlined]
│ [20] enzyme_call
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6144 [inlined]
│ [21] ForwardModeThunk
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6044 [inlined]
│ [22] autodiff
│ @ ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:680 [inlined]
│ [23] autodiff(mode::ForwardMode{false, FFIABI, false, false, false}, f::typeof(f), ::Type{Duplicated}, args::Duplicated{Float64})
│ @ Enzyme ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:549
│ [24] top-level scope
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:32
│ [25] include(mod::Module, _path::String)
│ @ Base ./Base.jl:557
│ [26] exec_options(opts::Base.JLOptions)
│ @ Base ./client.jl:323
│ [27] _start()
│ @ Base ./client.jl:531
└ @ Main ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:35
┌ Error: Reverse mode failed
│ exception =
│ EnzymeNoDerivativeError:
│ No augmented forward pass found for cuMemcpyHtoDAsync_v2
│ at context: %10 = call i32 @cuMemcpyHtoDAsync_v2(i64 %1, i64 %2, i64 8, i64 %9) #486, !dbg !995
│
│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424
│
│ Failure within method:
│
│ var"#unsafe_copyto!#1027"(::CuStream, ::Bool, ::typeof(unsafe_copyto!), ::CuPtr{Float64}, ::Ptr{Float64}, ::Int64)
│ @ CUDACore ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421
│
│ Hint: catch this exception as `err` and call `code_typed(err)` to inspect the surrounding code.
│
│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143 [inlined]
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529 [inlined]
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34 [inlined]
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35 [inlined]
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33 [inlined]
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424
│ [7] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421 [inlined]
│ [8] #1215
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:598 [inlined]
│ [9] #context!#1079
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:169 [inlined]
│ [10] context!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:164 [inlined]
│ [11] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:589
│ [12] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:547 [inlined]
│ [13] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:551 [inlined]
│ [14] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:439 [inlined]
│ [15] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:443 [inlined]
│ [16] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:452 [inlined]
│ [17] f
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:24 [inlined]
│ [18] diffejulia_f_383775wrap
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:0
│ [19] macro expansion
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6665 [inlined]
│ [20] enzyme_call
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6144 [inlined]
│ [21] CombinedAdjointThunk
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6028 [inlined]
│ [22] autodiff
│ @ ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:528 [inlined]
│ [23] autodiff(mode::ReverseMode{false, false, false, FFIABI, false, false}, f::typeof(f), ::Type{Active}, args::Active{Float64})
│ @ Enzyme ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:549
│ [24] top-level scope
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:40
│ [25] include(mod::Module, _path::String)
│ @ Base ./Base.jl:557
│ [26] exec_options(opts::Base.JLOptions)
│ @ Base ./client.jl:323
│ [27] _start()
│ @ Base ./client.jl:531
└ @ Main ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:43
```
Contributor guide
Assessment
This issue has not been assessed yet.