`test_ucx::test_transpose` fails with unclosed listener/endpoint error
- Dominant language
- Python
- Stars
- 1.7k
- Forks
- 778
- Avg merge
- 2h 50m
- Merged PRs (30d)
- 3
Description
```
@pytest.fixture(scope="function")
def ucx_loop():
"""Allows UCX to cancel progress tasks before closing event loop.
When UCX tasks are not completed in time (e.g., by unexpected Endpoint
closure), clean up tasks before closing the event loop to prevent unwanted
errors from being raised.
"""
ucp = pytest.importorskip("ucp")
loop = asyncio.new_event_loop()
loop.set_exception_handler(ucx_exception_handler)
ucp.reset()
yield loop
> ucp.reset()
distributed/utils_test.py:2125:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
def reset():
"""Resets the UCX library by shutting down all of UCX.
The library is initiated at next API call.
"""
global _ctx
if _ctx is not None:
weakref_ctx = weakref.ref(_ctx)
_ctx = None
gc.collect()
if weakref_ctx() is not None:
msg = (
"Trying to reset UCX but not all Endpoints and/or Listeners "
"are closed(). The following objects are still referencing "
"ApplicationContext: "
)
for o in gc.get_referrers(weakref_ctx()):
msg += "\n %s" % str(o)
> raise UCXError(msg)
E ucp._libs.exceptions.UCXError: Trying to reset UCX but not all Endpoints and/or Listeners are closed(). The following objects are still referencing ApplicationContext:
E
/opt/conda/envs/dask/lib/python3.9/site-packages/ucp/core.py:947: UCXError
```
https://gpuci.gpuopenanalytics.com/job/dask/job/distributed/job/prb/job/distributed-prb/5609/
Might be related to/duplicate of https://github.com/dask/distributed/issues/6222
cc @dask/gpu
Contributor guide
Assessment
This issue has not been assessed yet.