Crash on long prefix
- Dominant language
- Python
- Stars
- 2k
- Forks
- 561
- Avg merge
- 4d 5h
- Merged PRs (30d)
- 145
Description
```
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,804 INFO compiled_dag_node.py:2171 -- Tearing down compiled DAG
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,804 INFO compiled_dag_node.py:2176 -- Cancelling compiled worker on actor: Actor(RayWorkerWrapper, 75cedfa6dd0a507b40ef7cf810000000)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,804 INFO compiled_dag_node.py:2176 -- Cancelling compiled worker on actor: Actor(RayWorkerWrapper, f62bf6125779432420f1a59810000000)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,804 INFO compiled_dag_node.py:2176 -- Cancelling compiled worker on actor: Actor(RayWorkerWrapper, 5ada85deac242631df993ca410000000)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,804 INFO compiled_dag_node.py:2176 -- Cancelling compiled worker on actor: Actor(RayWorkerWrapper, 490a3989fe0ee432854bb86010000000)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,812 INFO compiled_dag_node.py:2198 -- Waiting for worker tasks to exit
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m 2025-12-14 15:30:24,812 INFO compiled_dag_node.py:2201 -- Teardown complete
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m Process EngineCore_DP0:
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m Traceback (most recent call last):
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/dag/compiled_dag_node.py", line 2529, in _execute_until
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m result = self._dag_output_fetcher.read(timeout)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/experimental/channel/common.py", line 312, in read
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m outputs = self._read_list(timeout)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/experimental/channel/common.py", line 403, in _read_list
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m raise e
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/experimental/channel/common.py", line 385, in _read_list
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m result = c.read(min(remaining_timeout, iteration_timeout))
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/experimental/channel/shared_memory_channel.py", line 776, in read
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m return self._channel_dict[self._resolve_actor_id()].read(timeout)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/experimental/channel/shared_memory_channel.py", line 612, in read
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m output = self._buffers[self._next_read_index].read(timeout)
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/experimental/channel/shared_memory_channel.py", line 480, in read
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ret = self._worker.get_objects(
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "/opt/ray_venvs/nemo_rl.models.generation.vllm.vllm_worker_async.VllmAsyncGenerationWorker/lib/python3.12/site-packages/ray/_private/worker.py", line 942, in get_objects
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ] = self.core_worker.get_objects(
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "python/ray/_raylet.pyx", line 3215, in ray._raylet.CoreWorker.get_objects
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m File "python/ray/includes/common.pxi", line 119, in ray._raylet.check_status
[36m(VllmAsyncGenerationWorker pid=1779282, ip=10.65.20.7)[0m [1;36m(EngineCore_DP0 pid=1780626)[0;0m ray.exceptions.RayChannelTimeoutError: System error: Timed out acquiring the read lock.
```
Contributor guide
Assessment
This issue has not been assessed yet.