NVIDIA / NVIDIA/cudf

[BUG] Difference in `groupby().all()` with `cudf.pandas` and string dtype

Open
#19,312 0 comments 0 reactions 0 assignees View on GitHub
bug Python
Dominant language
C++
Stars
9.8k
Forks
1.1k
Avg merge
3d 6m
Merged PRs (30d)
278

Description

**Describe the bug**

This test fails with `cudf.pandas` enabled, thanks to the grouper column being present in the output

**Steps/Code to reproduce bug**

```python
# file: test.py
import pandas as pd
import pandas._testing as tm
from pandas import DataFrame

import pandas as pd
import pandas._testing as tm
from pandas import DataFrame

def test_it():
reduction_func = "all"

dtype = pd.StringDtype()

expected_dtype, expected_value = dtype, pd.NA
expected_dtype = "bool"
expected_value = False if reduction_func == "any" else True
df = DataFrame({"a": ["x"], "b": [pd.NA]}, dtype=dtype)

result = df.groupby(df["a"]).all()
index = pd.Index(["x"], name="a", dtype=dtype)
expected = DataFrame(
{"b": expected_value}, index=index, dtype=expected_dtype
)
tm.assert_equal(result, expected)
```

Test with `python -m cudf.pandas -m pytest test.py` to see the failure

```
_______________________________________________________________________________________________________________________________________________________________________________________ test_it _______________________________________________________________________________________________________________________________________________________________________________________

func = , args = (, ( a b
a
x True True, b
a
x True), {}), kwargs = {}, disable_module_accelerator = , fast = False
fast_args = (<[AttributeError('Unusable type. Falling back to the slow object') raised in repr()] _Unusable object at 0x7f189f02f050>, ( a b
a
x True True, b
a
x True), {}), fast_kwargs = {}, slow_args = (, ( a b
a
x True True, b
a
x True), {}), slow_kwargs = {}

def _fast_slow_function_call(
func: Callable,
/,
*args,
**kwargs,
) -> Any:
"""
Call `func` with all `args` and `kwargs` converted to their
respective fast type. If that fails, call `func` with all
`args` and `kwargs` converted to their slow type.

Wrap the result in a fast-slow proxy if it is a type we know how
to wrap.
"""
from .module_accelerator import disable_module_accelerator

fast = False
try:
with nvtx.annotate(
"EXECUTE_FAST",
color=_CUDF_PANDAS_NVTX_COLORS["EXECUTE_FAST"],
domain="cudf_pandas",
):
fast_args, fast_kwargs = _fast_arg(args), _fast_arg(kwargs)
> result = func(*fast_args, **fast_kwargs)

/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:997:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:28: in call_operator
return fn(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <[AttributeError('Unusable type. Falling back to the slow object') raised in repr()] _Unusable object at 0x7f189f02f050>, args = ( a b
a
x True True, b
a
x True), kwds = {}

def __call__(self, *args: Any, **kwds: Any) -> Any:
> raise NotImplementedError(
"Fast implementation not available. "
"Falling back to the slow implementation"
)
E NotImplementedError: Fast implementation not available. Falling back to the slow implementation

/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:88: NotImplementedError

During handling of the above exception, another exception occurred:

def test_it():
reduction_func = "all"

dtype = pd.StringDtype()

expected_dtype, expected_value = dtype, pd.NA
expected_dtype = "bool"
expected_value = False if reduction_func == "any" else True
df = DataFrame({"a": ["x"], "b": [pd.NA]}, dtype=dtype)

result = df.groupby(df["a"]).all()
index = pd.Index(["x"], name="a", dtype=dtype)
expected = DataFrame(
{"b": expected_value}, index=index, dtype=expected_dtype
)
> tm.assert_equal(result, expected)

test.py:21:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:722: in __call__
result, _ = _fast_slow_function_call(
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:1049: in _fast_slow_function_call
result = func(*slow_args, **slow_kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

fn = , args = ( a b
a
x True True, b
a
x True), kwargs = {}

def call_operator(fn, args, kwargs):
> return fn(*args, **kwargs)
E AssertionError: DataFrame are different
E
E DataFrame shape mismatch
E [left]: (1, 2)
E [right]: (1, 1)

/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:28: AssertionError
=============================================================================================================================================================================== short test summary info ===============================================================================================================================================================================
FAILED test.py::test_it - AssertionError: DataFrame are different
```

**Expected behavior**

No error / match pandas

**Environment overview (please complete the following information)**

This is observed with pandas 2.2.3 and 2.3.1; CI is currently failing because of this new test from pandas 2.3.1

**Environment details**
Please run and paste the output of the `cudf/print_env.sh` script here, to gather any other relevant environment details

**Additional context**
Add any other context about the problem here.

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.