[BUG] Running tensor failed with KeyError raised by storage

**Describe the bug**

Running tensor failed with KeyError raised by storage.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
a = mt.zeros((200, 4000), chunk_size=100)
b = mt.zeros((4000, 10), chunk_size=100)

c = a.dot(b).T
c.execute()
```

```
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
[Previous line repeated 1 more time]
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
[Previous line repeated 2 more times]
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
[Previous line repeated 3 more times]
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
[Previous line repeated 4 more times]
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
[Previous line repeated 5 more times]
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
Task exception was never retrieved
future: exception=KeyError(('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c'))>
Traceback (most recent call last):
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
File "/Users/qinxuye/miniconda3/lib/python3.7/asyncio/tasks.py", line 630, in _wrap_awaitable
return (yield from awaitable.__await__())
[Previous line repeated 6 more times]
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 77, in wait
return await self._aio_task
File "/Users/qinxuye/Workspace/mars/mars/deploy/oscar/session.py", line 712, in _run_in_background
raise task_result.error.with_traceback(task_result.traceback)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/scheduling/worker/execution.py", line 248, in internal_run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/runner.py", line 104, in run_subtask
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 470, in run
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 362, in run
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/subtask/worker/processor.py", line 260, in _store_data
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/batch.py", line 135, in _async_batch
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 190, in batch_put
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 470, in request_quota_with_spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/handler.py", line 492, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 167, in spill
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 154, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/backends/pool.py", line 496, in send
File "/home/admin/work/_public-mars-0.7.1.zip/mars/oscar/api.py", line 118, in __on_receive__
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 323, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 326, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 329, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/core.py", line 293, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in get_spill_keys
File "/home/admin/work/_public-mars-0.7.1.zip/mars/services/storage/spill.py", line 101, in
KeyError: ('VbeKzQrjRWYhprJPt50JMtfL', '9d430a6c6c1b9f2237faf86492a17d6c')
```

Start by reproducing the tensor example, then trace the storage path through services/storage/handler.py and services/storage/spill.py, especially get_spill_keys and spill. Compare the resulting KeyError with the surrounding storage and subtask stack frames; done means the example completes without the storage KeyError and the failure is covered by an appropriate regression test.

Written by the indexing model from the issue text.