dask / dask/dask-ml

SimpleImputer fails on constant category value

Open
#749 1 comment 0 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
951
Forks
262
PR merge metrics
No merged PRs in 30d

Description

dask_ml's SimpleImputer, unlike its sklearn relative, fails on categorical dtypes:

```python
import numpy as np
import pandas as pd
import sklearn.impute as ski
import dask_ml.impute as dki

df = pd.DataFrame({'a': pd.Series(['test', np.nan], dtype='category')})

si = ski.SimpleImputer(strategy='constant', fill_value='missing')
si.fit_transform(df)
```

works as expected

```python
di = dki.SimpleImputer(strategy='constant', fill_value='missing')
di.fit_transform(df)
```

fails with traceback

```
ValueError Traceback (most recent call last)
in
1 di = dki.SimpleImputer(strategy='constant', fill_value='missing')
----> 2 di.fit_transform(df)

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/sklearn/base.py in fit_transform(self, X, y, **fit_params)
688 if y is None:
689 # fit method of arity 1 (unsupervised transformation)
--> 690 return self.fit(X, **fit_params).transform(X)
691 else:
692 # fit method of arity 2 (supervised transformation)

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/dask_ml/impute.py in transform(self, X)
82 def transform(self, X):
83 if isinstance(X, (pd.Series, pd.DataFrame, dd.Series, dd.DataFrame)):
---> 84 return X.fillna(self.statistics_)
85
86 elif isinstance(X, da.Array):

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/frame.py in fillna(self, value, method, axis, inplace, limit, downcast)
4315 downcast=None,
4316 ) -> Optional["DataFrame"]:
-> 4317 return super().fillna(
4318 value=value,
4319 method=method,

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/generic.py in fillna(self, value, method, axis, inplace, limit, downcast)
6072 continue
6073 obj = result[k]
-> 6074 obj.fillna(v, limit=limit, inplace=True, downcast=downcast)
6075 return result if not inplace else None
6076

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/series.py in fillna(self, value, method, axis, inplace, limit, downcast)
4515 downcast=None,
4516 ) -> Optional["Series"]:
-> 4517 return super().fillna(
4518 value=value,
4519 method=method,

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/generic.py in fillna(self, value, method, axis, inplace, limit, downcast)
6055 )
6056
-> 6057 new_data = self._mgr.fillna(
6058 value=value, limit=limit, inplace=inplace, downcast=downcast
6059 )

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/internals/managers.py in fillna(self, value, limit, inplace, downcast)
583
584 def fillna(self, value, limit, inplace: bool, downcast) -> "BlockManager":
--> 585 return self.apply(
586 "fillna", value=value, limit=limit, inplace=inplace, downcast=downcast
587 )

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/internals/managers.py in apply(self, f, align_keys, **kwargs)
404 applied = b.apply(f, **kwargs)
405 else:
--> 406 applied = getattr(b, f)(**kwargs)
407 result_blocks = _extend_blocks(applied, result_blocks)
408

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/internals/blocks.py in fillna(self, value, limit, inplace, downcast)
1777 def fillna(self, value, limit=None, inplace=False, downcast=None):
1778 values = self.values if inplace else self.values.copy()
-> 1779 values = values.fillna(value=value, limit=limit)
1780 return [
1781 self.make_block_same_class(

~/opt/anaconda3/envs/myenv/lib/python3.8/site-packages/pandas/core/arrays/categorical.py in fillna(self, value, method, limit)
1719 elif is_hashable(value):
1720 if not isna(value) and value not in self.categories:
-> 1721 raise ValueError("fill value must be in categories")
1722
1723 mask = codes == -1

ValueError: fill value must be in categories
```

**Environment**:

- Dask version: 1.7.0
- Python version: 3.8
- Operating System: MacOSX 10.14.3
- Install method (conda, pip, source): conda

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.