mars-project / mars-project/mars

[BUG] Unexpected error was raised when remote function returns Mars tensors

Open
#1,634 0 comments 0 reactions 0 assignees View on GitHub
mod: remote type: bug
Dominant language
Python
Stars
2.7k
Forks
325
PR merge metrics
No merged PRs in 30d

Description

**To Reproduce**
Create a Mars cluster and summit a remote function:

``` Python
In [47]: def func():
...: a = mt.random.rand(1000, 1000)
...: b = mt.random.rand(5000, 5000)
...: a.execute()
...: b.execute()
...: return a, b
...:

In [48]: print(mr.spawn(func).execute().fetch())
---------------------------------------------------------------------------
error Traceback (most recent call last)
/home/admin/work/public-mars-0.5.1.zip/mars/promise.py in _wrapped()

/home/admin/work/public-mars-0.5.1.zip/mars/worker/calc.py in ()

/home/admin/work/public-mars-0.5.1.zip/mars/worker/calc.py in _start_calc()

/home/admin/work/public-mars-0.5.1.zip/mars/utils.py in _wrapped()

/home/admin/work/public-mars-0.5.1.zip/mars/worker/calc.py in _calc_results()

/home/admin/work/public-mars-0.5.1.zip/mars/serialize/dataserializer.py in serialize()

/home/admin/work/public-mars-0.5.1.zip/mars/errors.py in __init__()

/home/admin/work/public-mars-0.5.1.zip/mars/core.py in __repr__()

/home/admin/work/public-mars-0.5.1.zip/mars/tensor/core.py in __repr__()

/home/admin/work/public-mars-0.5.1.zip/mars/tensor/core.py in _to_str()

/home/admin/work/public-mars-0.5.1.zip/mars/tensor/utils.py in fetch_corner_data()

/home/admin/work/public-mars-0.5.1.zip/mars/core.py in fetch()

/home/admin/work/public-mars-0.5.1.zip/mars/core.py in fetch()

/home/admin/work/public-mars-0.5.1.zip/mars/session.py in fetch()

/home/admin/work/public-mars-0.5.1.zip/mars/session.py in fetch()

/home/admin/work/public-mars-0.5.1.zip/mars/api.py in fetch_data()

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.ActorRef.__getattr__._mt_call()
64 else:
---> 65 return self.send((item,) + args + (kwargs,), wait=wait)
66

~/Documents/mars_dev/mars/mars/actors/core.pyx in mars.actors.core.ActorRef.send()
36 cpdef object send(self, object message, bint wait=True, object callback=None):
---> 37 return self._ctx.send(self, message, wait=wait, callback=callback)
38

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper.send()
604
--> 605 cpdef send(self, ActorRef actor_ref, object message, bint wait=True, object callback=None):
606 return self._send(actor_ref, message, wait_response=True, wait=wait, callback=callback)

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper.send()
605 cpdef send(self, ActorRef actor_ref, object message, bint wait=True, object callback=None):
--> 606 return self._send(actor_ref, message, wait_response=True, wait=wait, callback=callback)
607

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send()
599 if wait:
--> 600 return self._pool.apply(self._send_remote, (actor_ref.address, binaries))
601

/opt/conda/lib/python3.7/site-packages/gevent/pool.py in apply()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_cgreenlet.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_cgreenlet.Greenlet.get()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_cgreenlet.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_cgreenlet.Greenlet._raise_exception()

/opt/conda/lib/python3.7/site-packages/gevent/_compat.py in reraise()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_cgreenlet.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_cgreenlet.Greenlet.run()

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote()
469
--> 470 cpdef object _send_remote(self, str address, object binary):
471 cdef bytes res_binary

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote()
473
--> 474 with self._new_connection(address) as sock:
475 try:

~/Documents/mars_dev/mars/mars/actors/pool/gevent_pool.pyx in mars.actors.pool.gevent_pool.ActorRemoteHelper._send_remote()
476 write_remote_message(sock.sendall, *binary)
--> 477 res_binary = read_remote_message(sock.recv)
478 message_type = unpack_message_type(res_binary)

~/Documents/mars_dev/mars/mars/actors/pool/messages.pyx in mars.actors.pool.messages.read_remote_message()
846
--> 847 read_bytes = _wrap_read_func(read_func, 8)
848 size = _unpack_long(read_bytes)

~/Documents/mars_dev/mars/mars/actors/pool/messages.pyx in mars.actors.pool.messages._wrap_read_func()
829 try:
--> 830 read_bytes = read_func(size)
831 except ConnectionResetError:

/opt/conda/lib/python3.7/site-packages/gevent/_socket3.py in recv()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_hub_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_hub_primitives.wait_on_socket()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_hub_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_hub_primitives.wait_on_socket()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_hub_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_hub_primitives._primitive_wait()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_hub_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_hub_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_hub_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_hub_primitives.WaitOperationsGreenlet.wait()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_waiter.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_waiter.Waiter.get()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_greenlet_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_greenlet_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch()

/opt/conda/lib/python3.7/site-packages/gevent/_gevent_c_greenlet_primitives.cpython-37m-x86_64-linux-gnu.so in gevent._gevent_c_greenlet_primitives.SwitchOutGreenletWithLoop.switch()

src/gevent/_gevent_c_greenlet_primitives.pxd in gevent._gevent_c_greenlet_primitives._greenlet_switch()

error: cannot switch to a different thread

The above exception was the direct cause of the following exception:

ExecutionFailed Traceback (most recent call last)
in
----> 1 print(mr.spawn(func).execute().fetch())

~/Documents/mars_dev/mars/mars/core.py in execute(self, session, **kw)
373
374 if wait:
--> 375 return run()
376 else:
377 # leverage ThreadPoolExecutor to submit task,

~/Documents/mars_dev/mars/mars/core.py in run()
368 def run():
369 # no more fetch, thus just fire run
--> 370 session.run(self, **kw)
371 # return Tileable or ExecutableTuple itself
372 return self

~/Documents/mars_dev/mars/mars/session.py in run(self, *tileables, **kw)
498 tileables = tuple(mt.tensor(t) if not isinstance(t, (Entity, Base)) else t
499 for t in tileables)
--> 500 result = self._sess.run(*tileables, **kw)
501
502 for t in tileables:

~/Documents/mars_dev/mars/mars/web/session.py in run(self, *tileables, **kw)
212 timeout_val = min(check_interval, timeout - time_elapsed) if timeout > 0 else check_interval
213 try:
--> 214 if self._check_response_finished(graph_url, timeout_val):
215 break
216 except KeyboardInterrupt:

~/Documents/mars_dev/mars/mars/web/session.py in _check_response_finished(self, graph_url, timeout)
172 exc_info = pickle.loads(base64.b64decode(resp_json['exc_info']))
173 exc = exc_info[1].with_traceback(exc_info[2])
--> 174 raise ExecutionFailed('Graph execution failed.') from exc
175 else:
176 raise ExecutionFailed('Graph execution failed with unknown reason.')

ExecutionFailed: 'Graph execution failed.'

```

Contributor guide

Open the contributing guide

Research direction

Reproduce the failure with the remote function shown in the issue, then trace the tensor return through mars/serialize/dataserializer.py and mars/tensor/core.py. Compare the serialization and fetch paths with the actor messaging files named in the traceback; done means a remote function returning multiple Mars tensors completes without ExecutionFailed.

Written by the indexing model from the issue text.

Assessment

Tech stack
python
Domain
distributed-systems
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Stale
Clarity
Needs clarification
Newbie friendliness
25/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.