GIL-holding functions

Ubuntu 20.04.3 LTS
Python 3.8.10
dask-scheduler, version 2021.12.0
distributed, version 2021.12.0

distributed.core - INFO - Event loop was unresponsive in Worker for 7.66s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability.
```
#!/usr/bin/env python3

import xarray as xr
import numpy as np
from scipy.stats import weibull_min, norm
import dask
from dask.diagnostics import ProgressBar
import datetime
from os import cpu_count
import os

def weibull(data):
try:
paras = np.array(weibull_min.fit(data))
except:
paras = np.array([np.nan, np.nan, np.nan])
return paras

if __name__ == '__main__':

from distributed import Client
client = Client('192.168.0.13:8786')
#client = Client('10.10.10.13:8786')
print('client: ',client)

# method2
#CPUS = int(cpu_count()/2)
#client = Client(n_workers=CPUS, threads_per_worker=1)

# method3
#client = Client(n_workers=1, processes=True, threads_per_worker=5)

starttime = datetime.datetime.now()
'''
#f = xr.open_mfdataset(f'/lustre/data/prod/nc/r002/hourly/2019/*/ll_*_wsp.nc', parallel=True)
#f = xr.open_mfdataset(f'/lustre/data/prod/nc/r002/hourly/2019/*/ll_*_wsp.nc', parallel=True, chunks={'lev':1, 'lat': 100, 'lon': 100})
f = xr.open_mfdataset(f'/lustre/data/prod/nc/r002/hourly/2019/2019010*/ll_*_wsp.nc', parallel=True, chunks={'lev':3, 'lat': 100, 'lon': 100})
ntime, nlev, nlat, nlon = f.wsp.shape
data = f.wsp.chunk({'time': ntime, 'lev':1, 'lat': 100, 'lon': 100})
print(data.shape)
'''

nTime=120
rs = np.random.RandomState(0)
data = xr.DataArray(rs.randn(nTime, 18, 100, 100)+10, dims=["time","lev","lat","lon"])
data = data.chunk({'time':nTime, "lev":2, "lat":20, "lon":20})

endtime = datetime.datetime.now()
print('read data time:',endtime - starttime)

def work(x):
print(11)
result = xr.apply_ufunc(weibull,
x,
input_core_dims=[['time']],
dask='parallelized',
output_dtypes=[float],
vectorize=True,
output_core_dims=[['paras']],
dask_gufunc_kwargs={'output_sizes': {'paras': 3}}
)
return result

def save_nc(res):
res = res.rename('weibull_paras')
save = res.to_netcdf("/data/tmp/wsp_weibull_fit3.nc", compute=True)
#save = res.to_netcdf("/tmp/wsp_weibull_fit.nc", compute=True)
return save

print("weibull fit")
starttime = datetime.datetime.now()
print("scatter")
data_future = client.scatter(data)
print("submit")
res = client.submit(work, data_future)
print("result")
res = res.result()
#print(res)
endtime = datetime.datetime.now()
print('weibull fit time:',endtime - starttime)

print("save nc data")
starttime = datetime.datetime.now()
print("scatter")
res_future = client.scatter(res)
print("submit")
res2 = client.submit(save_nc, res_future)
print("result")
res2 = res2.result()

#res = res.rename('weibull_paras')
#save = res.to_netcdf("/opt/data/tmp/wsp_weibull_fit2.nc", compute=False)
#save.compute()

endtime = datetime.datetime.now()
print('save data time:',endtime - starttime)
```

```
distributed.core - INFO - Event loop was unresponsive in Worker for 9.39s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability.
distributed.core - INFO - Event loop was unresponsive in Worker for 10.88s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability.
distributed.utils - ERROR - Timed out trying to connect to tcp://192.168.0.13:34469 after 30 s
Traceback (most recent call last):
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/comm/core.py", line 284, in connect
comm = await asyncio.wait_for(
File "/usr/lib/python3.8/asyncio/tasks.py", line 501, in wait_for
raise exceptions.TimeoutError()
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/utils.py", line 681, in log_errors
yield
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/worker.py", line 1528, in close
await r.close_gracefully()
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/core.py", line 883, in send_recv_from_rpc
comm = await self.pool.connect(self.addr)
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/core.py", line 1071, in connect
raise exc
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/core.py", line 1055, in connect
comm = await fut
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/comm/core.py", line 308, in connect
raise OSError(
OSError: Timed out trying to connect to tcp://192.168.0.13:34469 after 30 s
tornado.application - ERROR - Exception in callback functools.partial(>, exception=OSError('Timed out trying to connect to tcp://192.168.0.13:34469 after 30 s')>)
Traceback (most recent call last):
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/comm/core.py", line 284, in connect
comm = await asyncio.wait_for(
File "/usr/lib/python3.8/asyncio/tasks.py", line 501, in wait_for
raise exceptions.TimeoutError()
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
File "/data/home/longrun/.local/lib/python3.8/site-packages/tornado/ioloop.py", line 741, in _run_callback
ret = callback()
File "/data/home/longrun/.local/lib/python3.8/site-packages/tornado/ioloop.py", line 765, in _discard_future_result
future.result()
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/worker.py", line 1293, in heartbeat
await self.close(report=False)
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/worker.py", line 1528, in close
await r.close_gracefully()
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/core.py", line 883, in send_recv_from_rpc
comm = await self.pool.connect(self.addr)
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/core.py", line 1071, in connect
raise exc
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/core.py", line 1055, in connect
comm = await fut
File "/data/home/longrun/.local/lib/python3.8/site-packages/distributed/comm/core.py", line 308, in connect
raise OSError(
OSError: Timed out trying to connect to tcp://192.168.0.13:34469 after 30 s
distributed.core - INFO - Event loop was unresponsive in Worker for 14.02s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability.
distributed.core - INFO - Event loop was unresponsive in Worker for 12.64s. This is often caused by long-running GIL-holding functions or moving large chunks of data. This can cause timeouts and instability.
```