NVIDIA / NVIDIA/cudf

[BUG] Higher memory usage when reading parquet in 26.04 vs 26.02

Open
#22,367 7 comments 0 reactions 0 assignees View on GitHub
bug cuIO no-oom Python
Dominant language
C++
Stars
9.8k
Forks
1.1k
Avg merge
3d 6m
Merged PRs (30d)
278

Description

**Describe the bug**
When reading a parquet files with the schema int64, list types I see a higher memory usage (~1GB more) on 26.04 vs 26.02.

**Steps/Code to reproduce bug**
```python
import threading
import time
from pathlib import Path

import cudf
import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pynvml

OUTPUT_DIR = Path("./synthetic_minhash")
NUM_FILES = 4
ROWS_PER_FILE = 500_000
MINHASHES_PER_ROW = 260
NUM_READS = 10
SEED = 42

def generate() -> list[str]:
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
files = []
for i in range(NUM_FILES):
path = OUTPUT_DIR / f"synthetic_{i:02d}.parquet"
if not path.exists():
rng = np.random.default_rng(SEED + i)
ids = np.arange(i * ROWS_PER_FILE, (i + 1) * ROWS_PER_FILE, dtype=np.int64)
flat = rng.integers(0, 2**32, size=ROWS_PER_FILE * MINHASHES_PER_ROW, dtype=np.uint32)
offsets = np.arange(0, (ROWS_PER_FILE + 1) * MINHASHES_PER_ROW, MINHASHES_PER_ROW, dtype=np.int32)
table = pa.table(
{
"id": pa.array(ids, type=pa.int64()),
"minhash": pa.ListArray.from_arrays(offsets, pa.array(flat, type=pa.uint32())),
}
)
pq.write_table(table, path, compression="snappy")
files.append(str(path))
return files

def main() -> None:
files = generate()

pynvml.nvmlInit()
handle = pynvml.nvmlDeviceGetHandleByIndex(0)
samples: list[tuple[float, int]] = []
stop = threading.Event()

def sampler() -> None:
while not stop.is_set():
samples.append((time.monotonic(), pynvml.nvmlDeviceGetMemoryInfo(handle).used))
time.sleep(0.005)

t = threading.Thread(target=sampler, daemon=True)
t.start()

overheads = []
for _ in range(NUM_READS):
rs = time.monotonic()
df = cudf.read_parquet(files, columns=["id", "minhash"])
rd = time.monotonic()
post = pynvml.nvmlDeviceGetMemoryInfo(handle).used
peak = max(u for ts, u in samples if rs <= ts <= rd)
overheads.append(peak - post)
del df

stop.set()
t.join()

steady = overheads[2:]
print(f"cudf {cudf.__version__}")
print(f"transient overhead avg (steady state): {sum(steady) / len(steady) / 1024**3:.2f} GiB")
print(f"peak GPU used overall: {max(u for _, u in samples) / 1024**3:.2f} GiB")

if __name__ == "__main__":
main()

```

**Expected behavior**
A clear and concise description of what you expected to happen.

**Environment overview (please complete the following information)**
- Environment location: [Bare-metal, Docker, Cloud(specify cloud provider)]
- Method of cuDF install: [conda, Docker, or from source]
- If method of install is [Docker], provide `docker pull` & `docker run` commands used

**Environment details**
Please run and paste the output of the `cudf/print_env.sh` script here, to gather any other relevant environment details

**Additional context**
Add any other context about the problem here.

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.