[BUG] Difference in `groupby().all()` with `cudf.pandas` and string dtype
- Dominant language
- C++
- Stars
- 9.8k
- Forks
- 1.1k
- Avg merge
- 3d 6m
- Merged PRs (30d)
- 278
Description
**Describe the bug**
This test fails with `cudf.pandas` enabled, thanks to the grouper column being present in the output
**Steps/Code to reproduce bug**
```python
# file: test.py
import pandas as pd
import pandas._testing as tm
from pandas import DataFrame
import pandas as pd
import pandas._testing as tm
from pandas import DataFrame
def test_it():
reduction_func = "all"
dtype = pd.StringDtype()
expected_dtype, expected_value = dtype, pd.NA
expected_dtype = "bool"
expected_value = False if reduction_func == "any" else True
df = DataFrame({"a": ["x"], "b": [pd.NA]}, dtype=dtype)
result = df.groupby(df["a"]).all()
index = pd.Index(["x"], name="a", dtype=dtype)
expected = DataFrame(
{"b": expected_value}, index=index, dtype=expected_dtype
)
tm.assert_equal(result, expected)
```
Test with `python -m cudf.pandas -m pytest test.py` to see the failure
```
_______________________________________________________________________________________________________________________________________________________________________________________ test_it _______________________________________________________________________________________________________________________________________________________________________________________
func = , args = (, ( a b
a
x True True, b
a
x True), {}), kwargs = {}, disable_module_accelerator = , fast = False
fast_args = (<[AttributeError('Unusable type. Falling back to the slow object') raised in repr()] _Unusable object at 0x7f189f02f050>, ( a b
a
x True True, b
a
x True), {}), fast_kwargs = {}, slow_args = (, ( a b
a
x True True, b
a
x True), {}), slow_kwargs = {}
def _fast_slow_function_call(
func: Callable,
/,
*args,
**kwargs,
) -> Any:
"""
Call `func` with all `args` and `kwargs` converted to their
respective fast type. If that fails, call `func` with all
`args` and `kwargs` converted to their slow type.
Wrap the result in a fast-slow proxy if it is a type we know how
to wrap.
"""
from .module_accelerator import disable_module_accelerator
fast = False
try:
with nvtx.annotate(
"EXECUTE_FAST",
color=_CUDF_PANDAS_NVTX_COLORS["EXECUTE_FAST"],
domain="cudf_pandas",
):
fast_args, fast_kwargs = _fast_arg(args), _fast_arg(kwargs)
> result = func(*fast_args, **fast_kwargs)
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:997:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:28: in call_operator
return fn(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <[AttributeError('Unusable type. Falling back to the slow object') raised in repr()] _Unusable object at 0x7f189f02f050>, args = ( a b
a
x True True, b
a
x True), kwds = {}
def __call__(self, *args: Any, **kwds: Any) -> Any:
> raise NotImplementedError(
"Fast implementation not available. "
"Falling back to the slow implementation"
)
E NotImplementedError: Fast implementation not available. Falling back to the slow implementation
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:88: NotImplementedError
During handling of the above exception, another exception occurred:
def test_it():
reduction_func = "all"
dtype = pd.StringDtype()
expected_dtype, expected_value = dtype, pd.NA
expected_dtype = "bool"
expected_value = False if reduction_func == "any" else True
df = DataFrame({"a": ["x"], "b": [pd.NA]}, dtype=dtype)
result = df.groupby(df["a"]).all()
index = pd.Index(["x"], name="a", dtype=dtype)
expected = DataFrame(
{"b": expected_value}, index=index, dtype=expected_dtype
)
> tm.assert_equal(result, expected)
test.py:21:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:722: in __call__
result, _ = _fast_slow_function_call(
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:1049: in _fast_slow_function_call
result = func(*slow_args, **slow_kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
fn = , args = ( a b
a
x True True, b
a
x True), kwargs = {}
def call_operator(fn, args, kwargs):
> return fn(*args, **kwargs)
E AssertionError: DataFrame are different
E
E DataFrame shape mismatch
E [left]: (1, 2)
E [right]: (1, 1)
/raid/toaugspurger/envs/rapidsai/cudf/25.08/lib/python3.12/site-packages/cudf/pandas/fast_slow_proxy.py:28: AssertionError
=============================================================================================================================================================================== short test summary info ===============================================================================================================================================================================
FAILED test.py::test_it - AssertionError: DataFrame are different
```
**Expected behavior**
No error / match pandas
**Environment overview (please complete the following information)**
This is observed with pandas 2.2.3 and 2.3.1; CI is currently failing because of this new test from pandas 2.3.1
**Environment details**
Please run and paste the output of the `cudf/print_env.sh` script here, to gather any other relevant environment details
**Additional context**
Add any other context about the problem here.
Contributor guide
Assessment
This issue has not been assessed yet.