dask / dask/dask-cloudprovider
AccessDeniedException from SageMaker
- Dominant language
- Python
- Stars
- 147
- Forks
- 119
- PR merge metrics
- No merged PRs in 30d
Description
When I try to launch a Fargate cluster from a SageMaker notebook, it fails with an AccessDeniedException as shown below. I don't know if there's a way round this.
```---------------------------------------------------------------------------
AccessDeniedException Traceback (most recent call last)
in ()
3 # })
4
----> 5 cluster = FargateCluster()
6
7 from distributed import Client
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/dask_cloudprovider/providers/aws/ecs.py in __init__(self, **kwargs)
1051
1052 def __init__(self, **kwargs):
-> 1053 super().__init__(fargate_scheduler=True, fargate_workers=True, **kwargs)
1054
1055
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/dask_cloudprovider/providers/aws/ecs.py in __init__(self, fargate_scheduler, fargate_workers, image, scheduler_cpu, scheduler_mem, scheduler_timeout, worker_cpu, worker_mem, worker_gpu, n_workers, cluster_arn, cluster_name_template, execution_role_arn, task_role_arn, task_role_policies, cloudwatch_logs_group, cloudwatch_logs_stream_prefix, cloudwatch_logs_default_retention, vpc, subnets, security_groups, environment, tags, skip_cleanup, **kwargs)
553 self._skip_cleanup = skip_cleanup
554 self._lock = asyncio.Lock()
--> 555 super().__init__(**kwargs)
556
557 async def _start(self,):
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/distributed/deploy/spec.py in __init__(self, workers, scheduler, worker, asynchronous, loop, security, silence_logs, name)
240 if not self.asynchronous:
241 self._loop_runner.start()
--> 242 self.sync(self._start)
243 self.sync(self._correct_state)
244
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/distributed/deploy/cluster.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
160 return future
161 else:
--> 162 return sync(self.loop, func, *args, **kwargs)
163
164 async def _logs(self, scheduler=True, workers=True):
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
332 if error[0]:
333 typ, exc, tb = error[0]
--> 334 raise exc.with_traceback(tb)
335 else:
336 return result[0]
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/distributed/utils.py in f()
316 if callback_timeout is not None:
317 future = gen.with_timeout(timedelta(seconds=callback_timeout), future)
--> 318 result[0] = yield future
319 except Exception as exc:
320 error[0] = sys.exc_info()
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/tornado/gen.py in run(self)
1097
1098 try:
-> 1099 value = future.result()
1100 except Exception:
1101 self.had_exception = True
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/dask_cloudprovider/providers/aws/ecs.py in _start(self)
569 self._skip_cleanup = self.config.get("skip_cleanup")
570 if not self._skip_cleanup:
--> 571 await _cleanup_stale_resources()
572
573 self._clients = await self._get_clients()
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/dask_cloudprovider/providers/aws/ecs.py in _cleanup_stale_resources()
1073 active_clusters = []
1074 clusters_to_delete = []
-> 1075 async for page in ecs.get_paginator("list_clusters").paginate():
1076 clusters = (
1077 await ecs.describe_clusters(
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/async_generator/_impl.py in step()
364 try:
365 self.ag_running = True
--> 366 return await ANextIter(self._it, start_fn, *args)
367 except StopAsyncIteration:
368 self._pypy_issue2786_workaround.discard(self._coroutine)
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/async_generator/_impl.py in __next__(self)
197 return self._invoke(first_fn, *first_args)
198 else:
--> 199 return self._invoke(self._it.__next__)
200
201 def send(self, value):
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/async_generator/_impl.py in _invoke(self, fn, *args)
207 def _invoke(self, fn, *args):
208 try:
--> 209 result = fn(*args)
210 except StopIteration as e:
211 # The underlying generator returned, so we should signal the end
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/aiobotocore/paginate.py in __anext__(self)
78
79 while True:
---> 80 response = await self._make_request(current_kwargs)
81 parsed = self._extract_parsed_response(response)
82 if first_request:
~/anaconda3/envs/pytorch_p36/lib/python3.6/site-packages/aiobotocore/client.py in _make_api_call(self, operation_name, api_params)
103 error_code = parsed_response.get("Error", {}).get("Code")
104 error_class = self.exceptions.from_code(error_code)
--> 105 raise error_class(parsed_response, operation_name)
106 else:
107 return parsed_response
AccessDeniedException: An error occurred (AccessDeniedException) when calling the ListClusters operation: User: arn:aws:sts::536099501702:assumed-role/AmazonSageMaker-ExecutionRole-20171130T095318/SageMaker is not authorized to perform: ecs:ListClusters on resource: *```
Contributor guide
Assessment
This issue has not been assessed yet.