test_as_completed_distributed timing out on windows

https://dev.azure.com/dask-dev/dask/_build/results?buildId=709&view=logs&j=7ba0bb95-26bc-52d3-29af-86059c433561&t=ac6d7656-d358-512c-6ebf-8527b90be0a5

```
2020-02-17T23:30:52.8388708Z ================================== FAILURES ===================================
2020-02-17T23:30:52.8388972Z ________________________ test_as_completed_distributed ________________________
2020-02-17T23:30:52.8389074Z
2020-02-17T23:30:52.8389186Z loop =
2020-02-17T23:30:52.8389293Z
2020-02-17T23:30:52.8389445Z def test_as_completed_distributed(loop): # noqa
2020-02-17T23:30:52.8389578Z cluster_kwargs = dict(active_rpc_timeout=10, nanny=Nanny)
2020-02-17T23:30:52.8389692Z if DISTRIBUTED_211:
2020-02-17T23:30:52.8389804Z cluster_kwargs["disconnect_timeout"] = 10
2020-02-17T23:30:52.8389930Z with cluster(**cluster_kwargs) as (s, [a, b]):
2020-02-17T23:30:52.8390066Z with Client(s["address"], loop=loop) as c:
2020-02-17T23:30:52.8390157Z counter_name = "counter_name"
2020-02-17T23:30:52.8390284Z counter = Variable(counter_name, client=c)
2020-02-17T23:30:52.8390403Z counter.set(0)
2020-02-17T23:30:52.8390509Z lock_name = "lock"
2020-02-17T23:30:52.8390615Z
2020-02-17T23:30:52.8390734Z killed_workers_name = "killed_workers"
2020-02-17T23:30:52.8390850Z killed_workers = Variable(killed_workers_name, client=c)
2020-02-17T23:30:52.8390977Z killed_workers.set({})
2020-02-17T23:30:52.8391083Z
2020-02-17T23:30:52.8391194Z X, y = make_classification(n_samples=100, n_features=10, random_state=0)
2020-02-17T23:30:52.8391324Z gs = dcv.GridSearchCV(
2020-02-17T23:30:52.8391445Z AsCompletedEstimator(killed_workers_name, lock_name, counter_name, 7),
2020-02-17T23:30:52.8391764Z param_grid={"foo_param": [0, 1, 2]},
2020-02-17T23:30:52.8391876Z cv=3,
2020-02-17T23:30:52.8391994Z refit=False,
2020-02-17T23:30:52.8392105Z cache_cv=False,
2020-02-17T23:30:52.8392225Z scheduler=c,
2020-02-17T23:30:52.8392335Z )
2020-02-17T23:30:52.8392441Z gs.fit(X, y)
2020-02-17T23:30:52.8392555Z
2020-02-17T23:30:52.8392660Z def f(dask_scheduler):
2020-02-17T23:30:52.8392781Z return dask_scheduler.transition_log
2020-02-17T23:30:52.8392889Z
2020-02-17T23:30:52.8392999Z def check_reprocess(transition_log):
2020-02-17T23:30:52.8393119Z finished = set()
2020-02-17T23:30:52.8393205Z for transition in transition_log:
2020-02-17T23:30:52.8393336Z key, start_state, end_state = (
2020-02-17T23:30:52.8393450Z transition[0],
2020-02-17T23:30:52.8393652Z transition[1],
2020-02-17T23:30:52.8393771Z transition[2],
2020-02-17T23:30:52.8393876Z )
2020-02-17T23:30:52.8393992Z assert key not in finished
2020-02-17T23:30:52.8394101Z if (
2020-02-17T23:30:52.8394192Z "score" in key
2020-02-17T23:30:52.8394292Z and start_state == "memory"
2020-02-17T23:30:52.8394413Z and end_state == "forgotten"
2020-02-17T23:30:52.8394529Z ):
2020-02-17T23:30:52.8394640Z finished.add(key)
2020-02-17T23:30:52.8394753Z
2020-02-17T23:30:52.8394864Z > check_reprocess(c.run_on_scheduler(f))
2020-02-17T23:30:52.8394956Z
2020-02-17T23:30:52.8395049Z tests\model_selection\dask_searchcv\test_model_selection.py:855:
2020-02-17T23:30:52.8395178Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2020-02-17T23:30:52.8395316Z C:\Miniconda\envs\dask-ml-test\lib\contextlib.py:119: in __exit__
2020-02-17T23:30:52.8395425Z next(self.gen)
2020-02-17T23:30:52.8395547Z C:\Miniconda\envs\dask-ml-test\lib\site-packages\distributed\utils_test.py:695: in cluster
2020-02-17T23:30:52.8395661Z lambda: disconnect_all(
2020-02-17T23:30:52.8395775Z C:\Miniconda\envs\dask-ml-test\lib\site-packages\tornado\ioloop.py:532: in run_sync
2020-02-17T23:30:52.8395896Z return future_cell[0].result()
2020-02-17T23:30:52.8396191Z C:\Miniconda\envs\dask-ml-test\lib\site-packages\distributed\utils_test.py:757: in disconnect_all
2020-02-17T23:30:52.8396308Z await asyncio.gather(*[disconnect(addr, timeout, rpc_kwargs) for addr in addresses])
2020-02-17T23:30:52.8396415Z C:\Miniconda\envs\dask-ml-test\lib\site-packages\distributed\utils_test.py:753: in disconnect
2020-02-17T23:30:52.8396526Z await asyncio.wait_for(do_disconnect(), timeout=timeout)
2020-02-17T23:30:52.8396642Z _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
2020-02-17T23:30:52.8396828Z
2020-02-17T23:30:52.8396930Z fut = .do_disconnect() done, defined at C:\Miniconda\envs\dask-ml-test\lib\site-packages\distributed\utils_test.py:748>>
2020-02-17T23:30:52.8397035Z timeout = 3
2020-02-17T23:30:52.8397106Z
2020-02-17T23:30:52.8397185Z async def wait_for(fut, timeout, *, loop=None):
2020-02-17T23:30:52.8397292Z """Wait for the single Future or coroutine to complete, with timeout.
2020-02-17T23:30:52.8397379Z
2020-02-17T23:30:52.8397569Z Coroutine will be wrapped in Task.
2020-02-17T23:30:52.8397683Z
2020-02-17T23:30:52.8397964Z Returns result of the Future or coroutine. When a timeout occurs,
2020-02-17T23:30:52.8398100Z it cancels the task and raises TimeoutError. To avoid the task
2020-02-17T23:30:52.8398222Z cancellation, wrap it in shield().
2020-02-17T23:30:52.8398409Z
2020-02-17T23:30:52.8398525Z If the wait is cancelled, the task is also cancelled.
2020-02-17T23:30:52.8398640Z
2020-02-17T23:30:52.8398757Z This function is a coroutine.
2020-02-17T23:30:52.8398871Z """
2020-02-17T23:30:52.8398989Z if loop is None:
2020-02-17T23:30:52.8399105Z loop = events.get_event_loop()
2020-02-17T23:30:52.8399219Z
2020-02-17T23:30:52.8399308Z if timeout is None:
2020-02-17T23:30:52.8399426Z return await fut
2020-02-17T23:30:52.8399547Z
2020-02-17T23:30:52.8399656Z if timeout <= 0:
2020-02-17T23:30:52.8399775Z fut = ensure_future(fut, loop=loop)
2020-02-17T23:30:52.8399901Z
2020-02-17T23:30:52.8400009Z if fut.done():
2020-02-17T23:30:52.8400132Z return fut.result()
2020-02-17T23:30:52.8400245Z
2020-02-17T23:30:52.8400353Z fut.cancel()
2020-02-17T23:30:52.8400551Z raise futures.TimeoutError()
2020-02-17T23:30:52.8400721Z
2020-02-17T23:30:52.8400822Z waiter = loop.create_future()
2020-02-17T23:30:52.8400953Z timeout_handle = loop.call_later(timeout, _release_waiter, waiter)
2020-02-17T23:30:52.8401077Z cb = functools.partial(_release_waiter, waiter)
2020-02-17T23:30:52.8401201Z
2020-02-17T23:30:52.8401314Z fut = ensure_future(fut, loop=loop)
2020-02-17T23:30:52.8401440Z fut.add_done_callback(cb)
2020-02-17T23:30:52.8401520Z
2020-02-17T23:30:52.8401606Z try:
2020-02-17T23:30:52.8401800Z # wait until the future completes or the timeout
2020-02-17T23:30:52.8401927Z try:
2020-02-17T23:30:52.8403299Z await waiter
2020-02-17T23:30:52.8403437Z except futures.CancelledError:
2020-02-17T23:30:52.8403570Z fut.remove_done_callback(cb)
2020-02-17T23:30:52.8403689Z fut.cancel()
2020-02-17T23:30:52.8403802Z raise
2020-02-17T23:30:52.8403940Z
2020-02-17T23:30:52.8404051Z if fut.done():
2020-02-17T23:30:52.8404173Z return fut.result()
2020-02-17T23:30:52.8404289Z else:
2020-02-17T23:30:52.8404404Z fut.remove_done_callback(cb)
2020-02-17T23:30:52.8404533Z # We must ensure that the task is not running
2020-02-17T23:30:52.8404653Z # after wait_for() returns.
2020-02-17T23:30:52.8404783Z # See https://bugs.python.org/issue32751
2020-02-17T23:30:52.8404908Z await _cancel_and_wait(fut, loop=loop)
2020-02-17T23:30:52.8405029Z > raise futures.TimeoutError()
2020-02-17T23:30:52.8405156Z E concurrent.futures._base.TimeoutError
2020-02-17T23:30:52.8405225Z
2020-02-17T23:30:52.8405325Z C:\Miniconda\envs\dask-ml-test\lib\asyncio\tasks.py:449: TimeoutError
2020-02-17T23:30:52.8405458Z ---------------------------- Captured stderr call -----------------------------
2020-02-17T23:30:52.8405604Z distributed.scheduler - INFO - Clear task state
2020-02-17T23:30:52.8405702Z
2020-02-17T23:30:52.8405819Z distributed.scheduler - INFO - Scheduler at: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8405927Z
2020-02-17T23:30:52.8406044Z distributed.nanny - INFO - Start Nanny at: 'tcp://127.0.0.1:2558'
2020-02-17T23:30:52.8406142Z
2020-02-17T23:30:52.8406322Z distributed.nanny - INFO - Start Nanny at: 'tcp://127.0.0.1:2567'
2020-02-17T23:30:52.8406421Z
2020-02-17T23:30:52.8406535Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2570
2020-02-17T23:30:52.8406645Z
2020-02-17T23:30:52.8406759Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2570
2020-02-17T23:30:52.8406856Z
2020-02-17T23:30:52.8406980Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8407079Z
2020-02-17T23:30:52.8407276Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8407463Z
2020-02-17T23:30:52.8407579Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8407689Z
2020-02-17T23:30:52.8407805Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8407904Z
2020-02-17T23:30:52.8408010Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-6fd6736b-94e9-472d-ab4c-b6ad05b2083c\worker-djj36pcz
2020-02-17T23:30:52.8408127Z
2020-02-17T23:30:52.8408222Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8408330Z
2020-02-17T23:30:52.8408522Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8408643Z
2020-02-17T23:30:52.8408760Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2570
2020-02-17T23:30:52.8408863Z
2020-02-17T23:30:52.8409049Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8409155Z
2020-02-17T23:30:52.8409245Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8409356Z
2020-02-17T23:30:52.8409445Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8409555Z
2020-02-17T23:30:52.8409667Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8409764Z
2020-02-17T23:30:52.8409862Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2576
2020-02-17T23:30:52.8409965Z
2020-02-17T23:30:52.8410054Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2576
2020-02-17T23:30:52.8410170Z
2020-02-17T23:30:52.8410257Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8410359Z
2020-02-17T23:30:52.8410485Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8410599Z
2020-02-17T23:30:52.8410700Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8410802Z
2020-02-17T23:30:52.8410892Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8411061Z
2020-02-17T23:30:52.8411185Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-902376fc-c51e-401b-a51c-cc7c180c23ee\worker-rzyb2rbr
2020-02-17T23:30:52.8411291Z
2020-02-17T23:30:52.8411474Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8411572Z
2020-02-17T23:30:52.8411680Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8411791Z
2020-02-17T23:30:52.8411976Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2576
2020-02-17T23:30:52.8412086Z
2020-02-17T23:30:52.8412197Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8412310Z
2020-02-17T23:30:52.8412409Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8412511Z
2020-02-17T23:30:52.8412629Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8412737Z
2020-02-17T23:30:52.8412911Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8413020Z
2020-02-17T23:30:52.8413110Z distributed.scheduler - INFO - Receive client connection: Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea
2020-02-17T23:30:52.8413221Z
2020-02-17T23:30:52.8413314Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8413416Z
2020-02-17T23:30:52.8413509Z distributed.scheduler - INFO - Receive client connection: Client-worker-764b391c-51dd-11ea-8660-000d3a9952ea
2020-02-17T23:30:52.8413606Z
2020-02-17T23:30:52.8413695Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8413806Z
2020-02-17T23:30:52.8413998Z distributed.scheduler - INFO - Receive client connection: Client-worker-764b391c-51dd-11ea-9214-000d3a9952ea
2020-02-17T23:30:52.8414103Z
2020-02-17T23:30:52.8414227Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8414324Z
2020-02-17T23:30:52.8414422Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8414548Z
2020-02-17T23:30:52.8414665Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2570
2020-02-17T23:30:52.8414765Z
2020-02-17T23:30:52.8414967Z distributed.scheduler - INFO - Remove client Client-worker-764b391c-51dd-11ea-9214-000d3a9952ea
2020-02-17T23:30:52.8415074Z
2020-02-17T23:30:52.8415176Z distributed.scheduler - INFO - Close client connection: Client-worker-764b391c-51dd-11ea-9214-000d3a9952ea
2020-02-17T23:30:52.8415287Z
2020-02-17T23:30:52.8415371Z distributed.nanny - INFO - Worker process 4628 exited with status 9
2020-02-17T23:30:52.8415549Z
2020-02-17T23:30:52.8415661Z distributed.nanny - WARNING - Restarting worker
2020-02-17T23:30:52.8415761Z
2020-02-17T23:30:52.8415868Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8415955Z
2020-02-17T23:30:52.8416077Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2576
2020-02-17T23:30:52.8416188Z
2020-02-17T23:30:52.8416299Z distributed.scheduler - INFO - Lost all workers
2020-02-17T23:30:52.8416407Z
2020-02-17T23:30:52.8416526Z distributed.scheduler - INFO - Remove client Client-worker-764b391c-51dd-11ea-8660-000d3a9952ea
2020-02-17T23:30:52.8416628Z
2020-02-17T23:30:52.8416756Z distributed.scheduler - INFO - Close client connection: Client-worker-764b391c-51dd-11ea-8660-000d3a9952ea
2020-02-17T23:30:52.8416861Z
2020-02-17T23:30:52.8416974Z distributed.nanny - INFO - Worker process 1632 exited with status 9
2020-02-17T23:30:52.8417093Z
2020-02-17T23:30:52.8417209Z distributed.nanny - WARNING - Restarting worker
2020-02-17T23:30:52.8417307Z
2020-02-17T23:30:52.8417431Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2609
2020-02-17T23:30:52.8417597Z
2020-02-17T23:30:52.8417709Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2609
2020-02-17T23:30:52.8417850Z
2020-02-17T23:30:52.8417946Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8418189Z
2020-02-17T23:30:52.8418360Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8418468Z
2020-02-17T23:30:52.8418568Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8418752Z
2020-02-17T23:30:52.8418880Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8419043Z
2020-02-17T23:30:52.8419139Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-902376fc-c51e-401b-a51c-cc7c180c23ee\worker-uplh9x8f
2020-02-17T23:30:52.8419325Z
2020-02-17T23:30:52.8419472Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8419608Z
2020-02-17T23:30:52.8419717Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2611
2020-02-17T23:30:52.8419820Z
2020-02-17T23:30:52.8419906Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2611
2020-02-17T23:30:52.8420019Z
2020-02-17T23:30:52.8420106Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8420210Z
2020-02-17T23:30:52.8420337Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8420435Z
2020-02-17T23:30:52.8420531Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8420634Z
2020-02-17T23:30:52.8420722Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8420928Z
2020-02-17T23:30:52.8421054Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-6fd6736b-94e9-472d-ab4c-b6ad05b2083c\worker-s5uqk0vu
2020-02-17T23:30:52.8421160Z
2020-02-17T23:30:52.8421285Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8421382Z
2020-02-17T23:30:52.8421515Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8421623Z
2020-02-17T23:30:52.8421738Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2609
2020-02-17T23:30:52.8421850Z
2020-02-17T23:30:52.8421933Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8422035Z
2020-02-17T23:30:52.8422133Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8422236Z
2020-02-17T23:30:52.8422358Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8422517Z
2020-02-17T23:30:52.8422604Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8422688Z
2020-02-17T23:30:52.8422792Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8422901Z
2020-02-17T23:30:52.8423001Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2611
2020-02-17T23:30:52.8423105Z
2020-02-17T23:30:52.8423270Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8423368Z
2020-02-17T23:30:52.8423458Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8423616Z
2020-02-17T23:30:52.8423735Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8423833Z
2020-02-17T23:30:52.8423951Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8424064Z
2020-02-17T23:30:52.8424156Z distributed.scheduler - INFO - Receive client connection: Client-worker-77e27a78-51dd-11ea-9318-000d3a9952ea
2020-02-17T23:30:52.8424276Z
2020-02-17T23:30:52.8424364Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8424466Z
2020-02-17T23:30:52.8424596Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8424702Z
2020-02-17T23:30:52.8424792Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2609
2020-02-17T23:30:52.8424896Z
2020-02-17T23:30:52.8424985Z distributed.scheduler - INFO - Remove client Client-worker-77e27a78-51dd-11ea-9318-000d3a9952ea
2020-02-17T23:30:52.8425263Z
2020-02-17T23:30:52.8425382Z distributed.scheduler - INFO - Close client connection: Client-worker-77e27a78-51dd-11ea-9318-000d3a9952ea
2020-02-17T23:30:52.8425488Z
2020-02-17T23:30:52.8425609Z distributed.nanny - INFO - Worker process 4888 exited with status 9
2020-02-17T23:30:52.8425724Z
2020-02-17T23:30:52.8425816Z distributed.nanny - WARNING - Restarting worker
2020-02-17T23:30:52.8425891Z
2020-02-17T23:30:52.8425980Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2624
2020-02-17T23:30:52.8426089Z
2020-02-17T23:30:52.8426203Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2624
2020-02-17T23:30:52.8426301Z
2020-02-17T23:30:52.8426393Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8426497Z
2020-02-17T23:30:52.8426612Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8426719Z
2020-02-17T23:30:52.8426904Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8427009Z
2020-02-17T23:30:52.8427123Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8427230Z
2020-02-17T23:30:52.8427414Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-902376fc-c51e-401b-a51c-cc7c180c23ee\worker-frt2hm40
2020-02-17T23:30:52.8427520Z
2020-02-17T23:30:52.8427606Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8427715Z
2020-02-17T23:30:52.8427921Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8428036Z
2020-02-17T23:30:52.8428152Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2624
2020-02-17T23:30:52.8428252Z
2020-02-17T23:30:52.8428367Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8428468Z
2020-02-17T23:30:52.8428557Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8428644Z
2020-02-17T23:30:52.8428733Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8428897Z
2020-02-17T23:30:52.8429017Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8429115Z
2020-02-17T23:30:52.8429216Z distributed.scheduler - INFO - Receive client connection: Client-worker-78c55b54-51dd-11ea-89f8-000d3a9952ea
2020-02-17T23:30:52.8429326Z
2020-02-17T23:30:52.8429438Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8429543Z
2020-02-17T23:30:52.8429665Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8429770Z
2020-02-17T23:30:52.8429861Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2611
2020-02-17T23:30:52.8429965Z
2020-02-17T23:30:52.8430056Z distributed.scheduler - INFO - Remove client Client-worker-78c55b54-51dd-11ea-89f8-000d3a9952ea
2020-02-17T23:30:52.8430168Z
2020-02-17T23:30:52.8430365Z distributed.scheduler - INFO - Close client connection: Client-worker-78c55b54-51dd-11ea-89f8-000d3a9952ea
2020-02-17T23:30:52.8430463Z
2020-02-17T23:30:52.8430556Z distributed.scheduler - INFO - Client Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea requests to retry 1 keys
2020-02-17T23:30:52.8430664Z
2020-02-17T23:30:52.8430787Z distributed.nanny - INFO - Worker process 2552 exited with status 9
2020-02-17T23:30:52.8430886Z
2020-02-17T23:30:52.8430970Z distributed.nanny - WARNING - Restarting worker
2020-02-17T23:30:52.8431079Z
2020-02-17T23:30:52.8431168Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2635
2020-02-17T23:30:52.8431244Z
2020-02-17T23:30:52.8431342Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2635
2020-02-17T23:30:52.8431421Z
2020-02-17T23:30:52.8431515Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8431615Z
2020-02-17T23:30:52.8431702Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8431821Z
2020-02-17T23:30:52.8431942Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8432039Z
2020-02-17T23:30:52.8432132Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8432239Z
2020-02-17T23:30:52.8432343Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-6fd6736b-94e9-472d-ab4c-b6ad05b2083c\worker-srnmp278
2020-02-17T23:30:52.8432451Z
2020-02-17T23:30:52.8432536Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8432642Z
2020-02-17T23:30:52.8432765Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8432871Z
2020-02-17T23:30:52.8432964Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2635
2020-02-17T23:30:52.8433067Z
2020-02-17T23:30:52.8433158Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8433387Z
2020-02-17T23:30:52.8433506Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8433612Z
2020-02-17T23:30:52.8433728Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8433825Z
2020-02-17T23:30:52.8434021Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8434120Z
2020-02-17T23:30:52.8434215Z distributed.scheduler - INFO - Receive client connection: Client-worker-79a7dd0c-51dd-11ea-93d0-000d3a9952ea
2020-02-17T23:30:52.8434330Z
2020-02-17T23:30:52.8434495Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8434595Z
2020-02-17T23:30:52.8434716Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8434797Z
2020-02-17T23:30:52.8434966Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2624
2020-02-17T23:30:52.8435133Z
2020-02-17T23:30:52.8435251Z distributed.scheduler - INFO - Remove client Client-worker-79a7dd0c-51dd-11ea-93d0-000d3a9952ea
2020-02-17T23:30:52.8435363Z
2020-02-17T23:30:52.8435543Z distributed.scheduler - INFO - Close client connection: Client-worker-79a7dd0c-51dd-11ea-93d0-000d3a9952ea
2020-02-17T23:30:52.8435648Z
2020-02-17T23:30:52.8435775Z distributed.scheduler - INFO - Client Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea requests to retry 1 keys
2020-02-17T23:30:52.8435878Z
2020-02-17T23:30:52.8435970Z distributed.nanny - INFO - Worker process 5072 exited with status 9
2020-02-17T23:30:52.8436072Z
2020-02-17T23:30:52.8436154Z distributed.nanny - WARNING - Restarting worker
2020-02-17T23:30:52.8436239Z
2020-02-17T23:30:52.8436420Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2647
2020-02-17T23:30:52.8436520Z
2020-02-17T23:30:52.8436620Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2647
2020-02-17T23:30:52.8436726Z
2020-02-17T23:30:52.8436847Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8436951Z
2020-02-17T23:30:52.8437039Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8437146Z
2020-02-17T23:30:52.8437260Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8437359Z
2020-02-17T23:30:52.8437546Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8437644Z
2020-02-17T23:30:52.8437739Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-902376fc-c51e-401b-a51c-cc7c180c23ee\worker-mctk4i60
2020-02-17T23:30:52.8437855Z
2020-02-17T23:30:52.8437971Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8438076Z
2020-02-17T23:30:52.8438169Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8438291Z
2020-02-17T23:30:52.8438484Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2647
2020-02-17T23:30:52.8438584Z
2020-02-17T23:30:52.8438670Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8438780Z
2020-02-17T23:30:52.8438941Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8439039Z
2020-02-17T23:30:52.8439163Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8439261Z
2020-02-17T23:30:52.8439455Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8439554Z
2020-02-17T23:30:52.8439734Z distributed.scheduler - INFO - Receive client connection: Client-worker-7a85fe10-51dd-11ea-815c-000d3a9952ea
2020-02-17T23:30:52.8439853Z
2020-02-17T23:30:52.8439964Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8440062Z
2020-02-17T23:30:52.8440257Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8440364Z
2020-02-17T23:30:52.8440490Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2635
2020-02-17T23:30:52.8440591Z
2020-02-17T23:30:52.8440756Z distributed.scheduler - INFO - Remove client Client-worker-7a85fe10-51dd-11ea-815c-000d3a9952ea
2020-02-17T23:30:52.8440866Z
2020-02-17T23:30:52.8440985Z distributed.scheduler - INFO - Close client connection: Client-worker-7a85fe10-51dd-11ea-815c-000d3a9952ea
2020-02-17T23:30:52.8441090Z
2020-02-17T23:30:52.8441221Z distributed.scheduler - INFO - Client Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea requests to retry 1 keys
2020-02-17T23:30:52.8441327Z
2020-02-17T23:30:52.8441509Z distributed.nanny - INFO - Worker process 348 exited with status 9
2020-02-17T23:30:52.8441608Z
2020-02-17T23:30:52.8441720Z distributed.nanny - WARNING - Restarting worker
2020-02-17T23:30:52.8441880Z
2020-02-17T23:30:52.8442005Z distributed.worker - INFO - Start worker at: tcp://127.0.0.1:2659
2020-02-17T23:30:52.8442082Z
2020-02-17T23:30:52.8442182Z distributed.worker - INFO - Listening to: tcp://127.0.0.1:2659
2020-02-17T23:30:52.8442283Z
2020-02-17T23:30:52.8442372Z distributed.worker - INFO - Waiting to connect to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8442572Z
2020-02-17T23:30:52.8442687Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8442785Z
2020-02-17T23:30:52.8442915Z distributed.worker - INFO - Threads: 1
2020-02-17T23:30:52.8443014Z
2020-02-17T23:30:52.8443108Z distributed.worker - INFO - Memory: 7.52 GB
2020-02-17T23:30:52.8443209Z
2020-02-17T23:30:52.8443381Z distributed.worker - INFO - Local Directory: D:\a\1\s\_test_worker-6fd6736b-94e9-472d-ab4c-b6ad05b2083c\worker-ul_kr4d_
2020-02-17T23:30:52.8443506Z
2020-02-17T23:30:52.8443628Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8443728Z
2020-02-17T23:30:52.8443836Z distributed.scheduler - INFO - Register worker
2020-02-17T23:30:52.8443947Z
2020-02-17T23:30:52.8444072Z distributed.scheduler - INFO - Starting worker compute stream, tcp://127.0.0.1:2659
2020-02-17T23:30:52.8444174Z
2020-02-17T23:30:52.8444289Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8444397Z
2020-02-17T23:30:52.8444513Z distributed.worker - INFO - Registered to: tcp://127.0.0.1:2555
2020-02-17T23:30:52.8444611Z
2020-02-17T23:30:52.8444706Z distributed.worker - INFO - -------------------------------------------------
2020-02-17T23:30:52.8444810Z
2020-02-17T23:30:52.8444892Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8445003Z
2020-02-17T23:30:52.8445129Z distributed.scheduler - INFO - Receive client connection: Client-worker-7b5f9834-51dd-11ea-92d8-000d3a9952ea
2020-02-17T23:30:52.8445253Z
2020-02-17T23:30:52.8445365Z distributed.core - INFO - Starting established connection
2020-02-17T23:30:52.8445463Z
2020-02-17T23:30:52.8445558Z distributed.worker - INFO - Run out-of-band function 'f'
2020-02-17T23:30:52.8445658Z
2020-02-17T23:30:52.8445746Z distributed.scheduler - INFO - Remove client Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea
2020-02-17T23:30:52.8445953Z
2020-02-17T23:30:52.8446080Z distributed.scheduler - INFO - Remove client Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea
2020-02-17T23:30:52.8446180Z
2020-02-17T23:30:52.8446309Z distributed.scheduler - INFO - Close client connection: Client-7489f8dc-51dd-11ea-8e18-000d3a9952ea
2020-02-17T23:30:52.8446411Z
2020-02-17T23:30:52.8446615Z distributed.nanny - INFO - Closing Nanny at 'tcp://127.0.0.1:2567'
2020-02-17T23:30:52.8446712Z
2020-02-17T23:30:52.8446797Z distributed.nanny - INFO - Closing Nanny at 'tcp://127.0.0.1:2558'
2020-02-17T23:30:52.8446969Z
2020-02-17T23:30:52.8447087Z distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:2659
2020-02-17T23:30:52.8447187Z
2020-02-17T23:30:52.8447284Z distributed.worker - INFO - Stopping worker at tcp://127.0.0.1:2647
2020-02-17T23:30:52.8447384Z
2020-02-17T23:30:52.8447479Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8447572Z
2020-02-17T23:30:52.8447690Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2659
2020-02-17T23:30:52.8447799Z
2020-02-17T23:30:52.8447921Z distributed.scheduler - INFO - Remove worker
2020-02-17T23:30:52.8448024Z
2020-02-17T23:30:52.8448149Z distributed.core - INFO - Removing comms to tcp://127.0.0.1:2647
2020-02-17T23:30:52.8448245Z
2020-02-17T23:30:52.8448326Z distributed.scheduler - INFO - Lost all workers
2020-02-17T23:30:52.8448565Z
2020-02-17T23:30:52.8448681Z distributed.nanny - INFO - Worker closed

```

Haven't debugged it yet.