JuliaDataCubes / JuliaDataCubes/YAXArrays.jl

open_mfdataset with a matrix of input arrays fails

Open
#261 1 comment 1 reaction 0 assignees View on GitHub

Nobody has claimed this yet.

Dominant language
Julia
Stars
132
Forks
25
PR merge metrics
No merged PRs in 30d

Description

When I have a matrix of input arrays and I want to open them with open_mfdataset, this fails with the following error:
My input data is perfectly aligned and they are non overlapping, but the check for Overlapping dimensions is checking every dimension separately and therefore finds overlaps.

ERROR: Dimension ranges overlap
Stacktrace:
  [1] error(s::String)
    @ Base ./error.jl:35
  [2] (::YAXArrays.Datasets.var"#ltfunc#277")(ax1::StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}, ax2::StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64})
    @ YAXArrays.Datasets ~/.julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:798
  [3] #1
    @ ./ordering.jl:133 [inlined]
  [4] lt
    @ ./ordering.jl:120 [inlined]
  [5] lt
    @ ./ordering.jl:118 [inlined]
  [6] lt
    @ ./ordering.jl:125 [inlined]
  [7] _sort!(v::Vector{Int64}, #unused#::Base.Sort.InsertionSortAlg, o::Base.Order.Perm{Base.Order.ReverseOrdering{Base.Order.Lt{Base.Order.var"#1#3"{YAXArrays.Datasets.var"#ltfunc#277", typeof(identity)}}}, Vector{StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}}, kw::NamedTuple{(:scratch, :lo, :hi), Tuple{Nothing, Int64, Int64}})
    @ Base.Sort ./sort.jl:748
  [8] _sort!(v::Vector{Int64}, a::Base.Sort.Small{10, Base.Sort.InsertionSortAlg, Base.Sort.IEEEFloatOptimization{Base.Sort.IsUIntMappable{Base.Sort.Small{40, Base.Sort.InsertionSortAlg, Base.Sort.CheckSorted{Base.Sort.ComputeExtrema{Base.Sort.ConsiderCountingSort{Base.Sort.CountingSort, Base.Sort.ConsiderRadixSort{Base.Sort.RadixSort, Base.Sort.Small{80, Base.Sort.InsertionSortAlg, Base.Sort.ScratchQuickSort{Missing, Missing, Base.Sort.InsertionSortAlg}}}}}}}, Base.Sort.StableCheckSorted{Base.Sort.ScratchQuickSort{Missing, Missing, Base.Sort.InsertionSortAlg}}}}}, o::Base.Order.Perm{Base.Order.ReverseOrdering{Base.Order.Lt{Base.Order.var"#1#3"{YAXArrays.Datasets.var"#ltfunc#277", typeof(identity)}}}, Vector{StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}}, kw::NamedTuple{(:scratch, :lo, :hi), Tuple{Nothing, Int64, Int64}})
    @ Base.Sort ./sort.jl:713
  [9] _sort!
    @ ./sort.jl:660 [inlined]
 [10] _sort!
    @ ./sort.jl:596 [inlined]
 [11] #sort!#28
    @ ./sort.jl:1374 [inlined]
 [12] sort!
    @ ./sort.jl:1367 [inlined]
 [13] _sortperm(A::Vector{StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}; alg::Base.Sort.MissingOptimization{Base.Sort.BoolOptimization{Base.Sort.Small{10, Base.Sort.InsertionSortAlg, Base.Sort.IEEEFloatOptimization{Base.Sort.IsUIntMappable{Base.Sort.Small{40, Base.Sort.InsertionSortAlg, Base.Sort.CheckSorted{Base.Sort.ComputeExtrema{Base.Sort.ConsiderCountingSort{Base.Sort.CountingSort, Base.Sort.ConsiderRadixSort{Base.Sort.RadixSort, Base.Sort.Small{80, Base.Sort.InsertionSortAlg, Base.Sort.ScratchQuickSort{Missing, Missing, Base.Sort.InsertionSortAlg}}}}}}}, Base.Sort.StableCheckSorted{Base.Sort.ScratchQuickSort{Missing, Missing, Base.Sort.InsertionSortAlg}}}}}}}, order::Base.Order.ReverseOrdering{Base.Order.Lt{Base.Order.var"#1#3"{YAXArrays.Datasets.var"#ltfunc#277", typeof(identity)}}}, scratch::Nothing, dims::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ Base.Sort ./sort.jl:1582
 [14] _sortperm
    @ ./sort.jl:1569 [inlined]
 [15] #sortperm#32
    @ ./sort.jl:1564 [inlined]
 [16] sortperm
    @ ./sort.jl:1555 [inlined]
 [17] analyse_axjoin_ranges(dimvallist::Vector{StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}})
    @ YAXArrays.Datasets ~/.julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:813
 [18] create_mergedict(dimvallist::Dict{Symbol, Vector{StepRangeLen{Float64, Base.TwicePrecision{Float64}, Base.TwicePrecision{Float64}, Int64}}})
    @ YAXArrays.Datasets ~/.julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:823
 [19] merge_datasets(dslist::Vector{Dataset})
    @ YAXArrays.Datasets ~/.julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:841
 [20] #open_mfdataset#49
    @ ~/.julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:259 [inlined]
 [21] open_mfdataset(g::Vector{String})
    @ YAXArrays.Datasets ~/.julia/dev/YAXArrays/src/DatasetAPI/Datasets.jl:259
 [22] top-level scope
    @ REPL[12]:1

Contributor guide

No contributing guide indexed for this repository

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Research direction

Start in src/DatasetAPI/Datasets.jl at open_mfdataset (line 259), then trace merge_datasets, create_mergedict, and analyse_axjoin_ranges around lines 798-841. Reproduce the failure with a matrix of aligned, non-overlapping input arrays and inspect the per-dimension overlap check. Done means open_mfdataset accepts this input without the Dimension ranges overlap error.

Written by the indexing model from the issue text.

Assessment

Tech stack
julia
Domain
data
Issue type
Bug
Difficulty
3/5
Estimated time
1-2 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
38/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.