EnzymeAD / EnzymeAD/Enzyme.jl

Error on differentiating through a function allocating CuArray (cuMemcpyHtoDAsync_v2)

Open
#3,064 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
Julia
Stars
586
Forks
108
Avg merge
1d 5h
Merged PRs (30d)
44

Description

The following example fails at forward or reverse mode since there is apparently no rule for the `cuMemcpyHtoDAsync_v2` CUDA function. Is this expected or is it a missing feature? Thanks!

```julia
using CUDA
using Enzyme

function f(θ)
h = [θ]
d = CuArray(h) # host-device copy (cuMemcpyHtoDAsync_v2)
return sum(d)
end

@info "primal" f(1.0) # 1.0

try
fwd, = Enzyme.autodiff(Forward, f, Duplicated, Duplicated(1.0, 1.0))
@info "Forward mode" fwd
catch e
@error "Forward mode failed" exception=(e, catch_backtrace())
end

try
rev, = Enzyme.autodiff(Reverse, f, Active, Active(1.0))[1]
@info "Reverse mode" rev
catch e
@error "Reverse mode failed" exception=(e, catch_backtrace())
end

```

Error stracktrace

```
┌ Info: primal
└ f(1.0) = 1.0

┌ Error: Forward mode failed
│ exception =
│ EnzymeNoDerivativeError:
│ No forward mode derivative found for cuMemcpyHtoDAsync_v2
│ at context: %10 = call i32 @cuMemcpyHtoDAsync_v2(i64 %1, i64 %2, i64 8, i64 %9) #480, !dbg !686

│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424

│ Failure within method:

│ var"#unsafe_copyto!#1027"(::CuStream, ::Bool, ::typeof(unsafe_copyto!), ::CuPtr{Float64}, ::Ptr{Float64}, ::Int64)
│ @ CUDACore ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421

│ Hint: catch this exception as `err` and call `code_typed(err)` to inspect the surrounding code.

│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143 [inlined]
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529 [inlined]
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34 [inlined]
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35 [inlined]
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33 [inlined]
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424
│ [7] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421 [inlined]
│ [8] #1215
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:598 [inlined]
│ [9] #context!#1079
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:169 [inlined]
│ [10] context!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:164 [inlined]
│ [11] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:589
│ [12] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:547 [inlined]
│ [13] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:551 [inlined]
│ [14] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:439 [inlined]
│ [15] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:443 [inlined]
│ [16] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:452 [inlined]
│ [17] f
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:24 [inlined]
│ [18] fwddiffejulia_f_347701wrap
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:0
│ [19] macro expansion
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6665 [inlined]
│ [20] enzyme_call
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6144 [inlined]
│ [21] ForwardModeThunk
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6044 [inlined]
│ [22] autodiff
│ @ ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:680 [inlined]
│ [23] autodiff(mode::ForwardMode{false, FFIABI, false, false, false}, f::typeof(f), ::Type{Duplicated}, args::Duplicated{Float64})
│ @ Enzyme ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:549
│ [24] top-level scope
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:32
│ [25] include(mod::Module, _path::String)
│ @ Base ./Base.jl:557
│ [26] exec_options(opts::Base.JLOptions)
│ @ Base ./client.jl:323
│ [27] _start()
│ @ Base ./client.jl:531
└ @ Main ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:35
┌ Error: Reverse mode failed
│ exception =
│ EnzymeNoDerivativeError:
│ No augmented forward pass found for cuMemcpyHtoDAsync_v2
│ at context: %10 = call i32 @cuMemcpyHtoDAsync_v2(i64 %1, i64 %2, i64 8, i64 %9) #486, !dbg !995

│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424

│ Failure within method:

│ var"#unsafe_copyto!#1027"(::CuStream, ::Bool, ::typeof(unsafe_copyto!), ::CuPtr{Float64}, ::Ptr{Float64}, ::Int64)
│ @ CUDACore ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421

│ Hint: catch this exception as `err` and call `code_typed(err)` to inspect the surrounding code.

│ Stacktrace:
│ [1] macro expansion
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:143 [inlined]
│ [2] macro expansion
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:529 [inlined]
│ [3] #60
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:34 [inlined]
│ [4] check
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/libcuda.jl:35 [inlined]
│ [5] cuMemcpyHtoDAsync_v2
│ @ ~/.julia/packages/GPUToolbox/x8hVh/src/ccalls.jl:33 [inlined]
│ [6] #unsafe_copyto!#1027
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:424
│ [7] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/memory.jl:421 [inlined]
│ [8] #1215
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:598 [inlined]
│ [9] #context!#1079
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:169 [inlined]
│ [10] context!
│ @ ~/.julia/packages/CUDACore/eDgUR/lib/cudadrv/state.jl:164 [inlined]
│ [11] unsafe_copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:589
│ [12] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:547 [inlined]
│ [13] copyto!
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:551 [inlined]
│ [14] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:439 [inlined]
│ [15] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:443 [inlined]
│ [16] CuArray
│ @ ~/.julia/packages/CUDACore/eDgUR/src/array.jl:452 [inlined]
│ [17] f
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:24 [inlined]
│ [18] diffejulia_f_383775wrap
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:0
│ [19] macro expansion
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6665 [inlined]
│ [20] enzyme_call
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6144 [inlined]
│ [21] CombinedAdjointThunk
│ @ ~/.julia/packages/Enzyme/C5rSn/src/compiler.jl:6028 [inlined]
│ [22] autodiff
│ @ ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:528 [inlined]
│ [23] autodiff(mode::ReverseMode{false, false, false, FFIABI, false, false}, f::typeof(f), ::Type{Active}, args::Active{Float64})
│ @ Enzyme ~/.julia/packages/Enzyme/C5rSn/src/Enzyme.jl:549
│ [24] top-level scope
│ @ ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:40
│ [25] include(mod::Module, _path::String)
│ @ Base ./Base.jl:557
│ [26] exec_options(opts::Base.JLOptions)
│ @ Base ./client.jl:323
│ [27] _start()
│ @ Base ./client.jl:531
└ @ Main ~/Workspace/tudelft1/WaterLilyADGPU.jl/enzyme_gpu_mwe.jl:43
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.