dask / dask/dask-kubernetes

Calling adapt multiple times results in an exception

Open
#634 1 comment 0 reactions 0 assignees View on GitHub
bug
Dominant language
Python
Stars
324
Forks
157
PR merge metrics
No merged PRs in 30d

Description

When a `KubeCluster` is already in adaptive mode calling `cluster.adapt()` again results in an error.

```python
cluster.adapt(4, 100)
---------------------------------------------------------------------------
ApiException Traceback (most recent call last)
~/Projects/dask/dask-kubernetes/dask_kubernetes/operator/kubecluster/kubecluster.py in _adapt(self, minimum, maximum)
631 try:
--> 632 await custom_objects_api.patch_namespaced_custom_object_scale(
633 group="kubernetes.dask.org",

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
191 e.body = e.body.decode('utf-8') if six.PY3 else e.body
--> 192 raise e
193

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
184 # perform request and return response
--> 185 response_data = await self.request(
186 method, url, query_params=query_params, headers=header_params,

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in PATCH(self, url, headers, query_params, post_params, body, _preload_content, _request_timeout)
250 _request_timeout=None):
--> 251 return (await self.request("PATCH", url,
252 headers=headers,

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
186 if not 200 <= r.status <= 299:
--> 187 raise ApiException(http_resp=r)
188

ApiException: (404)
Reason: Not Found
HTTP response headers:
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"daskautoscalers.kubernetes.dask.org \"simple\" not found","reason":"NotFound","details":{"name":"simple","group":"kubernetes.dask.org","kind":"daskautoscalers"},"code":404}

During handling of the above exception, another exception occurred:

ApiException Traceback (most recent call last)
in
----> 1 cluster.adapt(4, 100)

~/Projects/dask/dask-kubernetes/dask_kubernetes/operator/kubecluster/kubecluster.py in adapt(self, minimum, maximum)
621 >>> cluster.adapt(minimum=1, maximum=10) # Allow scheduler to add/remove workers within 1-10 range
622 """
--> 623 return self.sync(self._adapt, minimum, maximum)
624
625 async def _adapt(self, minimum=None, maximum=None):

~/Projects/dask/distributed/distributed/utils.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
337 return future
338 else:
--> 339 return sync(
340 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
341 )

~/Projects/dask/distributed/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
404 if error:
405 typ, exc, tb = error
--> 406 raise exc.with_traceback(tb)
407 else:
408 return result

~/Projects/dask/distributed/distributed/utils.py in f()
377 future = asyncio.wait_for(future, callback_timeout)
378 future = asyncio.ensure_future(future)
--> 379 result = yield future
380 except Exception:
381 error = sys.exc_info()

~/.local/lib/python3.8/site-packages/tornado/gen.py in run(self)
760
761 try:
--> 762 value = future.result()
763 except Exception:
764 exc_info = sys.exc_info()

~/Projects/dask/dask-kubernetes/dask_kubernetes/operator/kubecluster/kubecluster.py in _adapt(self, minimum, maximum)
639 )
640 except kubernetes.client.ApiException:
--> 641 await custom_objects_api.create_namespaced_custom_object(
642 group="kubernetes.dask.org",
643 version="v1",

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
190 except ApiException as e:
191 e.body = e.body.decode('utf-8') if six.PY3 else e.body
--> 192 raise e
193
194 self.last_response = response_data

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/api_client.py in __call_api(self, resource_path, method, path_params, query_params, header_params, body, post_params, files, response_types_map, auth_settings, _return_http_data_only, collection_formats, _preload_content, _request_timeout, _host, _request_auth)
183 try:
184 # perform request and return response
--> 185 response_data = await self.request(
186 method, url, query_params=query_params, headers=header_params,
187 post_params=post_params, body=body,

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in POST(self, url, headers, query_params, post_params, body, _preload_content, _request_timeout)
228 post_params=None, body=None, _preload_content=True,
229 _request_timeout=None):
--> 230 return (await self.request("POST", url,
231 headers=headers,
232 query_params=query_params,

~/.local/lib/python3.8/site-packages/kubernetes_asyncio/client/rest.py in request(self, method, url, query_params, headers, body, post_params, _preload_content, _request_timeout)
185
186 if not 200 <= r.status <= 299:
--> 187 raise ApiException(http_resp=r)
188
189 return r

ApiException: (409)
Reason: Conflict
HTTP response headers:
HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"daskautoscalers.kubernetes.dask.org \"simple\" already exists","reason":"AlreadyExists","details":{"name":"simple","group":"kubernetes.dask.org","kind":"daskautoscalers"},"code":409}
```

I would expect this to patch the existing autoscaler.

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.