as_completed(futures, with_results=True, raise_errors=raise_errors) gets struck in an infinite loop

Set up: Dask-gateway on AKS.

We are running multiple applications simultaneously from Ubuntu which create dask clusters. All the applications using dask invoke the as_completed method. Sometimes the as_completed method gets in a infinite loop. Log is below.

```
[2022-11-17 20:12:13,045] {logging_mixin.py:115} WARNING - 2022-11-17 20:12:13,014 - tornado.application - ERROR - Exception in callback functools.partial(>, exception=CommClosedError('in : Stream is closed')>)
Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/comm/tcp.py", line 225, in read
frames_nbytes = await stream.read_bytes(fmt_size)
tornado.iostream.StreamClosedError: Stream is closed

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/home/airflow/.local/lib/python3.8/site-packages/tornado/ioloop.py", line 741, in _run_callback
ret = callback()
File "/home/airflow/.local/lib/python3.8/site-packages/tornado/ioloop.py", line 765, in _discard_future_result
future.result()
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 4871, in _track_future
result = await future._result(raiseit=False)
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 299, in _result
result = await self.client._gather([self])
File "/home/airflow/.local/lib/python3.8/site-packages/distributed/client.py", line 2108, in _gather
response = await self._gather_future
```