JuliaParallel / JuliaParallel/DistributedArrays.jl
Broadcasting is broken with Darrays having differing ndims
- Dominant language
- Julia
- Stars
- 205
- Forks
- 34
- PR merge metrics
- No merged PRs in 30d
Description
```julia
julia> d = dzeros((3,3))
3×3 DArray{Float64,2,Array{Float64,2}}:
0.0 0.0 0.0
0.0 0.0 0.0
0.0 0.0 0.0
julia> d2 = dzeros((3,3,1))
3×3×1 DArray{Float64,3,Array{Float64,3}}:
[:, :, 1] =
0.0 0.0 0.0
0.0 0.0 0.0
0.0 0.0 0.0
julia> d .+ d2
ERROR: MethodError: no method matching Base.Broadcast.BroadcastStyle(::Type{Base.Broadcast.DefaultArrayStyle{2}}, ::Type{Base.Broadcast.DefaultArrayStyle{3}})
Closest candidates are:
Base.Broadcast.BroadcastStyle(::Type{T}) where T at broadcast.jl:103
Stacktrace:
[1] Base.Broadcast.BroadcastStyle(::DistributedArrays.DArrayStyle{Base.Broadcast.DefaultArrayStyle{2}}, ::DistributedArrays.DArrayStyle{Base.Broadcast.DefaultArrayStyle{3}}) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:21
[2] result_style(::DistributedArrays.DArrayStyle{Base.Broadcast.DefaultArrayStyle{2}}, ::DistributedArrays.DArrayStyle{Base.Broadcast.DefaultArrayStyle{3}}) at ./broadcast.jl:445
[3] combine_styles(::DArray{Float64,2,Array{Float64,2}}, ::DArray{Float64,3,Array{Float64,3}}) at ./broadcast.jl:421
[4] broadcasted(::Function, ::DArray{Float64,2,Array{Float64,2}}, ::DArray{Float64,3,Array{Float64,3}}) at ./broadcast.jl:1263
[5] top-level scope at REPL[79]:1
```
Another one with `reshape`:
```julia
julia> d .+ reshape(d, size(d)..., 1)
ERROR: TaskFailedException:
setindex! not defined for DArray{Float64,3,Array{Float64,3}}
error(::String, ::Type{T} where T) at ./error.jl:42
error_if_canonical_setindex(::IndexCartesian, ::DArray{Float64,3,Array{Float64,3}}, ::Int64, ::Int64, ::Int64) at ./abstractarray.jl:1163
setindex! at ./abstractarray.jl:1152 [inlined]
_setindex! at ./abstractarray.jl:1183 [inlined]
setindex! at ./abstractarray.jl:1153 [inlined]
macro expansion at ./multidimensional.jl:772 [inlined]
macro expansion at ./cartesian.jl:64 [inlined]
macro expansion at ./multidimensional.jl:767 [inlined]
_unsafe_getindex!(::DArray{Float64,3,Array{Float64,3}}, ::Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}, ::UnitRange{Int64}, ::UnitRange{Int64}, ::UnitRange{Int64}) at ./multidimensional.jl:762
_unsafe_getindex(::IndexCartesian, ::Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}, ::UnitRange{Int64}, ::UnitRange{Int64}, ::UnitRange{Int64}) at ./multidimensional.jl:757
_getindex(::IndexCartesian, ::Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}, ::UnitRange{Int64}, ::UnitRange{Int64}, ::Vararg{UnitRange{Int64},N} where N) at ./multidimensional.jl:743
getindex(::Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}, ::UnitRange{Int64}, ::UnitRange{Int64}, ::UnitRange{Int64}) at ./abstractarray.jl:1060
(::DistributedArrays.var"#72#74"{Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}},Array{Tuple{UnitRange{Int64},UnitRange{Int64},UnitRange{Int64}},3}})(::CartesianIndex{3}) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:548
localpart(::DistributedArrays.DestinationSerializer) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/serialize.jl:83
#73 at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:550 [inlined]
construct_localparts(::DistributedArrays.var"#73#75"{DistributedArrays.DestinationSerializer}, ::Tuple{Int64,Int64}, ::Tuple{Int64,Int64,Int64}, ::Array{Int64,3}, ::Array{Tuple{UnitRange{Int64},UnitRange{Int64},UnitRange{Int64}},3}, ::Array{Array{Int64,1},1}; T::Nothing, A::Nothing) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:118
construct_localparts at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:118 [inlined]
#137 at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:354 [inlined]
run_work_thunk(::Distributed.var"#137#138"{typeof(DistributedArrays.construct_localparts),Tuple{DistributedArrays.var"#73#75"{DistributedArrays.DestinationSerializer},Tuple{Int64,Int64},Tuple{Int64,Int64,Int64},Array{Int64,3},Array{Tuple{UnitRange{Int64},UnitRange{Int64},UnitRange{Int64}},3},Array{Array{Int64,1},1}},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}}, ::Bool) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/process_messages.jl:79
remotecall_fetch(::Function, ::Distributed.LocalProcess, ::Function, ::Vararg{Any,N} where N; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:379
remotecall_fetch(::Function, ::Distributed.LocalProcess, ::Function, ::Vararg{Any,N} where N) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:379
remotecall_fetch(::Function, ::Int64, ::Function, ::Vararg{Any,N} where N; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:421
remotecall_fetch at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:421 [inlined]
macro expansion at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:88 [inlined]
(::DistributedArrays.var"#1#3"{Tuple{Int64,Int64},DistributedArrays.var"#73#75"{DistributedArrays.DestinationSerializer},Tuple{Int64,Int64,Int64},Array{Int64,3},Array{Tuple{UnitRange{Int64},UnitRange{Int64},UnitRange{Int64}},3},Array{Array{Int64,1},1},Array{DataType,1},Int64})() at ./task.jl:356
Stacktrace:
[1] remotecall_fetch(::Function, ::Distributed.LocalProcess, ::Function, ::Vararg{Any,N} where N; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:380
[2] remotecall_fetch(::Function, ::Distributed.LocalProcess, ::Function, ::Vararg{Any,N} where N) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:379
[3] remotecall_fetch(::Function, ::Int64, ::Function, ::Vararg{Any,N} where N; kwargs::Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}) at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:421
[4] remotecall_fetch at /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.5/Distributed/src/remotecall.jl:421 [inlined]
[5] macro expansion at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:88 [inlined]
[6] (::DistributedArrays.var"#1#3"{Tuple{Int64,Int64},DistributedArrays.var"#73#75"{DistributedArrays.DestinationSerializer},Tuple{Int64,Int64,Int64},Array{Int64,3},Array{Tuple{UnitRange{Int64},UnitRange{Int64},UnitRange{Int64}},3},Array{Array{Int64,1},1},Array{DataType,1},Int64})() at ./task.jl:356
Stacktrace:
[1] sync_end(::Channel{Any}) at ./task.jl:314
[2] macro expansion at ./task.jl:333 [inlined]
[3] DArray(::Tuple{Int64,Int64}, ::Function, ::Tuple{Int64,Int64,Int64}, ::Array{Int64,3}, ::Array{Tuple{UnitRange{Int64},UnitRange{Int64},UnitRange{Int64}},3}, ::Array{Array{Int64,1},1}) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:83
[4] DArray(::Function, ::Tuple{Int64,Int64,Int64}, ::Array{Int64,1}, ::Array{Int64,1}) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:177
[5] distribute(::Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}; procs::Array{Int64,1}, dist::Array{Int64,1}) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:550
[6] distribute(::Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}) at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/darray.jl:543
[7] _bcdistribute at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:133 [inlined]
[8] bcdistribute at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:129 [inlined]
[9] bcdistribute_args at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:137 [inlined]
[10] bcdistribute_args at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:136 [inlined]
[11] bcdistribute at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:126 [inlined]
[12] copy at /home/jishnu/.julia/packages/DistributedArrays/3FSDt/src/broadcast.jl:93 [inlined]
[13] materialize(::Base.Broadcast.Broadcasted{DistributedArrays.DArrayStyle{Base.Broadcast.DefaultArrayStyle{2}},Nothing,typeof(+),Tuple{DArray{Float64,2,Array{Float64,2}},Base.ReshapedArray{Float64,3,DArray{Float64,2,Array{Float64,2}},Tuple{Base.MultiplicativeInverses.SignedMultiplicativeInverse{Int64}}}}}) at ./broadcast.jl:837
[14] top-level scope at REPL[80]:1
```
Contributor guide
No contributing guide indexed for this repository
Research direction
Start by reproducing both failures from the examples, then inspect the broadcast paths in src/broadcast.jl and the DArray construction and distribution paths in src/darray.jl named by the stack traces. Done means broadcasting works between Darrays with differing dimensions, including the reshape case, without MethodError or setindex! failures.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- julia
- Domain
- distributed-systems
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 42/100