apache / apache/arrow-julia

Test failures with CategoricalArrays 1.0

Open
#586 2 comments 0 reactions 0 assignees View on GitHub
Dominant language
Julia
Stars
312
Forks
78
PR merge metrics
No merged PRs in 30d

Description

I havne't been able to replicate it in CI, but I have replicated it on both a M1 Mac and and an x86-64 Ubuntu machine locally:

`BoundsError: attempt to access 3-element CategoricalArrays.CategoricalRefPool{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalPool{String, UInt32}} with indices 0:2 at index [3]`

cc @nalimilan

```julia
# some dict encoding coverage: Error During Test at /home/ubuntu/arrow-julia/test/runtests.jl:339
Got exception outside of a @test
BoundsError: attempt to access 3-element CategoricalArrays.CategoricalRefPool{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalPool{String, UInt32}} with indices 0:2 at index [3]
Stacktrace:
[1] throw_boundserror(A::CategoricalArrays.CategoricalRefPool{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalPool{String, UInt32}}, I::Tuple{Int64})
@ Base ./abstractarray.jl:737
[2] checkbounds
@ ./abstractarray.jl:702 [inlined]
[3] getindex
@ ~/.julia/packages/CategoricalArrays/ptzwC/src/array.jl:1161 [inlined]
[4] getindex
@ ~/.julia/packages/ArrowTypes/IVpuP/src/ArrowTypes.jl:445 [inlined]
[5] iterate
@ ./abstractarray.jl:1216 [inlined]
[6] Arrow.ValidityBitmap(x::ToArrow{Union{Missing, String}, CategoricalArrays.CategoricalRefPool{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalPool{String, UInt32}}})
@ Arrow ~/arrow-julia/src/arraytypes/arraytypes.jl:228
[7] arrowvector(::ListKind{true}, x::ToArrow{Union{Missing, String}, CategoricalArrays.CategoricalRefPool{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalPool{String, UInt32}}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; largelists::Bool, kw::@Kwargs{dictencode::Bool, maxdepth::Int64, dictencodenested::Bool, compression::Nothing})
@ Arrow ~/arrow-julia/src/arraytypes/list.jl:226
[8] arrowvector
@ ~/arrow-julia/src/arraytypes/list.jl:224 [inlined]
[9] #arrowvector#10
@ ~/arrow-julia/src/arraytypes/arraytypes.jl:150 [inlined]
[10] arrowvector
@ ~/arrow-julia/src/arraytypes/arraytypes.jl:148 [inlined]
[11] arrowvector(x::CategoricalArrays.CategoricalRefPool{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalPool{String, UInt32}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::@Kwargs{dictencodenested::Bool, compression::Nothing})
@ Arrow ~/arrow-julia/src/arraytypes/arraytypes.jl:113
[12] arrowvector(::DictEncodedKind, x::Arrow.DictEncode{Union{Missing, CategoricalValue{String, UInt32}}, CategoricalVector{Union{Missing, String}, UInt32, String, CategoricalValue{String, UInt32}, Missing}}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Base.ImmutableDict{String, String}; dictencode::Bool, dictencodenested::Bool, kw::@Kwargs{maxdepth::Int64, compression::Nothing})
@ Arrow ~/arrow-julia/src/arraytypes/dictencoding.jl:236
[13] arrowvector
@ ~/arrow-julia/src/arraytypes/dictencoding.jl:202 [inlined]
[14] #arrowvector#10
@ ~/arrow-julia/src/arraytypes/arraytypes.jl:150 [inlined]
[15] arrowvector
@ ~/arrow-julia/src/arraytypes/arraytypes.jl:148 [inlined]
[16] arrowvector(x::CategoricalVector{Union{Missing, String}, UInt32, String, CategoricalValue{String, UInt32}, Missing}, i::Int64, nl::Int64, fi::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; dictencoding::Bool, dictencode::Bool, maxdepth::Int64, kw::@Kwargs{compression::Nothing})
@ Arrow ~/arrow-julia/src/arraytypes/arraytypes.jl:113
[17] arrowvector
@ ~/arrow-julia/src/arraytypes/arraytypes.jl:68 [inlined]
[18] toarrowvector(x::CategoricalVector{Union{Missing, String}, UInt32, String, CategoricalValue{String, UInt32}, Missing}, i::Int64, de::Dict{Int64, Any}, ded::Vector{Arrow.DictEncoding}, meta::Nothing; compression::Nothing, kw::@Kwargs{})
@ Arrow ~/arrow-julia/src/arraytypes/arraytypes.jl:47
[19] toarrowvector
@ ~/arrow-julia/src/arraytypes/arraytypes.jl:36 [inlined]
[20] toarrowvector(x::CategoricalVector{Union{Missing, String}, UInt32, String, CategoricalValue{String, UInt32}, Missing})
@ Arrow ~/arrow-julia/src/arraytypes/arraytypes.jl:36
[21] macro expansion
@ ~/arrow-julia/test/runtests.jl:347 [inlined]
[22] macro expansion
@ ~/.julia/juliaup/julia-1.10.10+0.x64.linux.gnu/share/julia/stdlib/v1.10/Test/src/Test.jl:1577 [inlined]
[23] macro expansion
@ ~/arrow-julia/test/runtests.jl:341 [inlined]
[24] macro expansion
@ ~/.julia/juliaup/julia-1.10.10+0.x64.linux.gnu/share/julia/stdlib/v1.10/Test/src/Test.jl:1577 [inlined]
[25] macro expansion
@ ~/arrow-julia/test/runtests.jl:115 [inlined]
[26] macro expansion
@ ~/.julia/juliaup/julia-1.10.10+0.x64.linux.gnu/share/julia/stdlib/v1.10/Test/src/Test.jl:1577 [inlined]
[27] top-level scope
@ ~/arrow-julia/test/runtests.jl:54
[28] include(fname::String)
@ Base.MainInclude ./client.jl:494
[29] top-level scope
@ none:6
[30] eval
@ ./boot.jl:385 [inlined]
[31] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:296
[32] _start()
@ Base ./client.jl:557
```

Contributor guide

No contributing guide indexed for this repository

Research direction

Start with the dict encoding coverage around test/runtests.jl:339-347 and inspect the CategoricalArrays access reported at src/arraytypes/arraytypes.jl:228, along with the ArrowTypes frame in the stack trace. Reproduce locally with CategoricalArrays 1.0 and confirm the relevant tests complete without the BoundsError.

Written by the indexing model from the issue text.

Assessment

Tech stack
julia
Domain
data
Issue type
Bug
Difficulty
3/5
Estimated time
1-2 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.