NVIDIA / NVIDIA/cudf

Concatenated rows exceeds size_type range after merge operation[BUG]

Open
#8,182 4 comments 0 reactions 0 assignees View on GitHub
bug dask Python
Dominant language
C++
Stars
9.8k
Forks
1.1k
Avg merge
3d 6m
Merged PRs (30d)
278

Description

**Describe the bug**
Hi Guys, I got an error about `Total number of concatenated rows exceeds size_type range` after doing an`inner join` on two `dask_cudf` dfs. It seems that some partitions contains a large number of rows. However, the code works good when I `concat` these two dfs.

**Steps/Code to reproduce bug**
In my understanding, the rows in q2 should be larger than q1 .
```
# setup
c = LocalCUDACluster( device_memory_limit=0.8, rmm_managed_memory=True, jit_unspill=True)
c = Client(c)

# 40G data
a = dask_cudf.read_orc('a/*.orc')
# 4G data
b = dask_cudf.read_orc('b/*.orc')

# Works good
q2 = dask_cudf.concat([a,b])
q2.map_partitions(len).compute()

0 307200
1 291840
2 337920
3 261120
4 256000
...
21529 100525
21530 94142
21531 86762
21532 94782
21533 12502
Length: 21534, dtype: int64

# Errors with concatenated rows exceeds size_type range
q1 = a.merge(b, on=['a'],how='inner' )
length_partition = q1.map_partitions(len)
length_partition.compute()

---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
in
1 length_partition = q1.map_partitions(len)
----> 2 length_partition.compute()

/conda/envs/rapids/lib/python3.7/site-packages/dask/base.py in compute(self, **kwargs)
282 dask.base.compute
283 """
--> 284 (result,) = compute(self, traverse=False, **kwargs)
285 return result
286

/conda/envs/rapids/lib/python3.7/site-packages/dask/base.py in compute(*args, **kwargs)
564 postcomputes.append(x.__dask_postcompute__())
565
--> 566 results = schedule(dsk, keys, **kwargs)
567 return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
568

/conda/envs/rapids/lib/python3.7/site-packages/distributed/client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
2664 should_rejoin = False
2665 try:
-> 2666 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
2667 finally:
2668 for f in futures.values():

/conda/envs/rapids/lib/python3.7/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
1979 direct=direct,
1980 local_worker=local_worker,
-> 1981 asynchronous=asynchronous,
1982 )
1983

/conda/envs/rapids/lib/python3.7/site-packages/distributed/client.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
842 else:
843 return sync(
--> 844 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
845 )
846

/conda/envs/rapids/lib/python3.7/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
351 if error[0]:
352 typ, exc, tb = error[0]
--> 353 raise exc.with_traceback(tb)
354 else:
355 return result[0]

/conda/envs/rapids/lib/python3.7/site-packages/distributed/utils.py in f()
334 if callback_timeout is not None:
335 future = asyncio.wait_for(future, callback_timeout)
--> 336 result[0] = yield future
337 except Exception as exc:
338 error[0] = sys.exc_info()

/conda/envs/rapids/lib/python3.7/site-packages/tornado/gen.py in run(self)
760
761 try:
--> 762 value = future.result()
763 except Exception:
764 exc_info = sys.exc_info()

/conda/envs/rapids/lib/python3.7/site-packages/distributed/client.py in _gather(self, futures, errors, direct, local_worker)
1838 exc = CancelledError(key)
1839 else:
-> 1840 raise exception.with_traceback(traceback)
1841 raise exc
1842 if errors == "skip":

/conda/envs/rapids/lib/python3.7/site-packages/dask/dataframe/core.py in _concat()
101 args[0]
102 if not args2
--> 103 else methods.concat(args2, uniform=True, ignore_index=ignore_index)
104 )
105

/conda/envs/rapids/lib/python3.7/site-packages/dask/dataframe/methods.py in concat()
434 filter_warning=filter_warning,
435 ignore_index=ignore_index,
--> 436 **kwargs
437 )
438

/conda/envs/rapids/lib/python3.7/site-packages/dask_cuda/proxy_object.py in wrapper()
708 args = [unproxy(d) for d in args]
709 kwargs = {k: unproxy(v) for k, v in kwargs.items()}
--> 710 return func(*args, **kwargs)
711
712 return wrapper

/conda/envs/rapids/lib/python3.7/site-packages/dask/dataframe/methods.py in concat()
434 filter_warning=filter_warning,
435 ignore_index=ignore_index,
--> 436 **kwargs
437 )
438

/conda/envs/rapids/lib/python3.7/site-packages/dask_cudf/backends.py in concat_cudf()
223 )
224
--> 225 return cudf.concat(dfs, axis=axis, ignore_index=ignore_index)
226
227

/conda/envs/rapids/lib/python3.7/site-packages/cudf/core/reshape.py in concat()
370 ignore_index=ignore_index,
371 # Explicitly cast rather than relying on None being falsy.
--> 372 sort=bool(sort),
373 )
374 return result

/conda/envs/rapids/lib/python3.7/contextlib.py in inner()
72 def inner(*args, **kwds):
73 with self._recreate_cm():
---> 74 return func(*args, **kwds)
75 return inner
76

/conda/envs/rapids/lib/python3.7/site-packages/cudf/core/frame.py in _concat()
454 # Concatenate the Tables
455 out = cls._from_table(
--> 456 libcudf.concat.concat_tables(tables, ignore_index=ignore_index)
457 )
458

cudf/_lib/concat.pyx in cudf._lib.concat.concat_tables()

cudf/_lib/concat.pyx in cudf._lib.concat.concat_tables()

RuntimeError: cuDF failure at: ../src/copying/concatenate.cu:364: Total number of concatenated rows exceeds size_type range

```

**Environment overview (please complete the following information)**
- Environment location: Docker
- Method of cuDF install: conda

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.