JuliaDataCubes / JuliaDataCubes/YAXArrays.jl
Cube from map of cubes with different chunk sizes can't be mapcubed
Nobody has claimed this yet.
- Dominant language
- Julia
- Stars
- 132
- Forks
- 25
- PR merge metrics
- No merged PRs in 30d
Description
When I compute the difference of two cubes with map from cubes with different chunksizes, the computation is done correctly and I can access the data but I can't use mapCube afterwards anymore. Tried to save the cube and to construct a CubeTable.
wcordiff = map(wcor, wcorann) do x,y
x - y
end
julia> YAXArrays.Cubes.cubechunks(wcor)
(5820, 4290, 1)
julia> YAXArrays.Cubes.cubechunks(wcorann)
(1000, 1000, 1)
I get the following error, when I try to save the resulting cube:
julia> savecube(wcordiff, "data/cubes/wcordiff.zarr")
ERROR: Chunks do not align in dimension 1
Stacktrace:
[1] error(s::String)
@ Base ./error.jl:33
[2] (::DiskArrays.var"#52#60"{Vector{Tuple{Tuple{Int64, Int64, Int64}, Tuple{Int64, Int64, Int64}}}})(n::Int64)
@ DiskArrays ~/.julia/packages/DiskArrays/tD6J0/src/ops.jl:59
[3] ntuple
@ ./ntuple.jl:19 [inlined]
[4] common_chunks(::Tuple{Int64, Int64, Int64}, ::ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}, ::Vararg{ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}})
@ DiskArrays ~/.julia/packages/DiskArrays/tD6J0/src/ops.jl:55
[5] eachchunk
@ ~/.julia/packages/DiskArrays/tD6J0/src/ops.jl:26 [inlined]
[6] cubechunks(c::YAXArray{Union{Missing, Float32}, 3, DiskArrays.BroadcastDiskArray{Union{Missing, Float32}, 3, Base.Broadcast.Broadcasted{DiskArrays.ChunkStyle{3}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}, Base.OneTo{Int64}}, var"#6#7", Tuple{ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}, ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}}}}, Vector{CubeAxis}})
@ YAXArrays.Cubes ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/Cubes/Cubes.jl:150
[7] (::YAXArrays.DAT.var"#137#140"{YAXArrays.DAT.DATConfig{1, 1}, Dict{Int64, Int64}, Vector{NamedTuple{(:iloopax, :cs, :iscompressed, :innerleap, :preventpar), Tuple{Int64, Int64, Bool, Int64, Bool}}}})(lax::RangeAxis{Float64, :X, StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}, ilax::Int64)
@ YAXArrays.DAT ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/DAT/DAT.jl:883
[8] foreach(::Function, ::Vector{CubeAxis}, ::UnitRange{Int64})
@ Base ./abstractarray.jl:2695
[9] getCacheSizes(dc::YAXArrays.DAT.DATConfig{1, 1}, loopchunksizes::Dict{Any, Any})
@ YAXArrays.DAT ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/DAT/DAT.jl:875
[10] mapCube(::YAXArrays.DAT.var"#cop#269", ::Tuple{YAXArray{Union{Missing, Float32}, 3, DiskArrays.BroadcastDiskArray{Union{Missing, Float32}, 3, Base.Broadcast.Broadcasted{DiskArrays.ChunkStyle{3}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}, Base.OneTo{Int64}}, var"#6#7", Tuple{ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}, ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}}}}, Vector{CubeAxis}}}; max_cache::Float64, indims::InDims, outdims::OutDims, inplace::Bool, ispar::Bool, debug::Bool, include_loopvars::Bool, showprog::Bool, nthreads::Vector{Int64}, loopchunksize::Dict{Any, Any}, kwargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ YAXArrays.DAT ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/DAT/DAT.jl:431
[11] #mapCube#38
@ ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/DAT/DAT.jl:281 [inlined]
[12] savecube(c::YAXArray{Union{Missing, Float32}, 3, DiskArrays.BroadcastDiskArray{Union{Missing, Float32}, 3, Base.Broadcast.Broadcasted{DiskArrays.ChunkStyle{3}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}, Base.OneTo{Int64}}, var"#6#7", Tuple{ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}, ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}}}}, Vector{CubeAxis}}, name::String; chunksize::Dict{Any, Any}, max_cache::Float64, backend::Symbol, backendargs::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ YAXArrays.DAT ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/DAT/CubeIO.jl:53
[13] savecube(c::YAXArray{Union{Missing, Float32}, 3, DiskArrays.BroadcastDiskArray{Union{Missing, Float32}, 3, Base.Broadcast.Broadcasted{DiskArrays.ChunkStyle{3}, Tuple{Base.OneTo{Int64}, Base.OneTo{Int64}, Base.OneTo{Int64}}, var"#6#7", Tuple{ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}, ZArray{Union{Missing, Float32}, 3, Zarr.BloscCompressor, DirectoryStore}}}}, Vector{CubeAxis}}, name::String)
@ YAXArrays.DAT ~/Documents/papers_wip/EMDAmazonas/dev/YAXArrays/src/DAT/CubeIO.jl:17
[14] top-level scope
@ REPL[29]:1
For now I can circumvent the problem by constructing a new cube, but I am not sure, whether this is the best approach:
julia> newdiff = YAXArray(wcordiff.axes, wcordiff[:,:,:])
Contributor guide
No contributing guide indexed for this repository
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Start with the reproducer using map, savecube, and mapCube, then inspect YAXArrays/src/Cubes/Cubes.jl, YAXArrays/src/DAT/DAT.jl, and YAXArrays/src/DAT/CubeIO.jl around the reported stack frames. Check how DiskArrays common_chunks handles the differing chunk sizes. Done means a cube produced from the subtraction can be saved and passed to mapCube without the chunk-alignment error.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- julia
- Domain
- data-engineering
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 42/100