[BUG] Execution failed due to plasma full

Nobody has claimed this yet.

**Describe the bug**

Execution failed due to plasma full.

**To Reproduce**
To help us reproducing this bug, please provide information below:
1. Your Python version
2. The version of Mars you use
3. Versions of crucial packages, such as numpy, scipy and pandas
4. Full stack of the error.
5. Minimized code to reproduce the error.

```
2021-08-18 14:27:11,005 mars.services.task.supervisor.stage 47 ERROR Subtask ULpoVOujpCSbu8SQsavXGOtK errored
Traceback (most recent call last):
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/scheduling/worker/execution.py", line 304, in internal_run_subtask
subtask, band_name, subtask_api, batch_quota_req)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/scheduling/worker/execution.py", line 384, in _retry_run_subtask
return await _run_subtask_once()
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/scheduling/worker/execution.py", line 359, in _run_subtask_once
return await asyncio.shield(aiotask)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/subtask/api.py", line 59, in run_subtask_in_slot
return await ref.run_subtask(subtask)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/context.py", line 154, in send
return self._process_result_message(result)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
raise message.error.with_traceback(message.traceback)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/pool.py", line 496, in send
result = await future
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/api.py", line 118, in __on_receive__
return await super().__on_receive__(message)
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/subtask/worker/runner.py", line 102, in run_subtask
result = yield self._running_processor.run(subtask)
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/context.py", line 154, in send
return self._process_result_message(result)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
raise message.error.with_traceback(message.traceback)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/pool.py", line 496, in send
result = await future
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/api.py", line 118, in __on_receive__
return await super().__on_receive__(message)
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 345, in mars.oscar.core._BaseActor.__on_receive__
return await self._handle_actor_result(result)
File "mars/oscar/core.pyx", line 250, in _handle_actor_result
result = list(dones)[0].result()
File "mars/oscar/core.pyx", line 273, in mars.oscar.core._BaseActor._run_actor_async_generator
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 275, in mars.oscar.core._BaseActor._run_actor_async_generator
async with self._lock:
File "mars/oscar/core.pyx", line 279, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await gen.athrow(*res)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/subtask/worker/processor.py", line 457, in run
result = yield self._running_aio_task
File "mars/oscar/core.pyx", line 284, in mars.oscar.core._BaseActor._run_actor_async_generator
res = await self._handle_actor_result(res)
File "mars/oscar/core.pyx", line 219, in _handle_actor_result
result = await result
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/subtask/worker/processor.py", line 352, in run
stored_keys, store_sizes, memory_sizes = await self._store_data(chunk_graph)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/subtask/worker/processor.py", line 250, in _store_data
store_infos = await put_infos
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/batch.py", line 135, in _async_batch
return await self.batch_func(args_list, kwargs_list)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/storage/api/oscar.py", line 115, in batch_put
return await self._storage_handler_ref.put.batch(*puts)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/context.py", line 154, in send
return self._process_result_message(result)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/context.py", line 59, in _process_result_message
raise message.error.with_traceback(message.traceback)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/backends/pool.py", line 496, in send
result = await future
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/api.py", line 118, in __on_receive__
return await super().__on_receive__(message)
File "mars/oscar/core.pyx", line 351, in __on_receive__
raise ex
File "mars/oscar/core.pyx", line 332, in mars.oscar.core._BaseActor.__on_receive__
with debug_async_timeout('actor_lock_timeout',
File "mars/oscar/core.pyx", line 335, in mars.oscar.core._BaseActor.__on_receive__
async with self._lock:
File "mars/oscar/core.pyx", line 341, in mars.oscar.core._BaseActor.__on_receive__
result = await result
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/oscar/batch.py", line 135, in _async_batch
return await self.batch_func(args_list, kwargs_list)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/services/storage/handler.py", line 195, in batch_put
object_info = await self._clients[level].put(obj)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/storage/plasma.py", line 241, in put
await f.write(buffer)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/lib/aio/base.py", line 25, in method
return await self._loop.run_in_executor(self._executor, func)
File "/opt/conda/lib/python3.7/concurrent/futures/thread.py", line 57, in run
result = self.fn(*self.args, **self.kwargs)
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/storage/plasma.py", line 66, in write
self._write_init(
File "/home/admin/work/_cupid_test_release-pymars-0.8.0a1-js.zip/mars/storage/plasma.py", line 55, in _write_init
self._buffer = buf = self._plasma_client.create(self._object_id, self._size)
File "pyarrow/_plasma.pyx", line 366, in pyarrow._plasma.PlasmaClient.create
plasma_check_status(
File "pyarrow/_plasma.pyx", line 286, in pyarrow._plasma.plasma_check_status
raise PlasmaStoreFull(message)
pyarrow._plasma.PlasmaStoreFull: object does not fit in the plasma store
```

Start with mars/storage/plasma.py, especially put and write, then follow the call from mars/services/storage/handler.py:195 through batch_put. First collect the Python, Mars, package versions, full stack, and minimized code requested in the issue, then reproduce the PlasmaStoreFull failure. Done means the cause is isolated and the execution no longer fails due to the plasma store being full.

Written by the indexing model from the issue text.