decode error

PD separation deployment, benchmark test concurrency 2028 error

[2025-09-18 20:03:54 DP13 TP13] TpModelWorkerClient hit an exception: Traceback (most recent call last):
File "/sgl-workspace/sglang/python/sglang/srt/managers/tp_worker_overlap_thread.py", line 129, in forward_thread_func
self.forward_thread_func_()
File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/managers/tp_worker_overlap_thread.py", line 164, in forward_thread_func_
self.worker.forward_batch_generation(
File "/sgl-workspace/sglang/python/sglang/srt/managers/tp_worker.py", line 214, in forward_batch_generation
logits_output, can_run_cuda_graph = self.model_runner.forward(
File "/sgl-workspace/sglang/python/sglang/srt/model_executor/model_runner.py", line 1472, in forward
output = self._forward_raw(
File "/sgl-workspace/sglang/python/sglang/srt/model_executor/model_runner.py", line 1499, in _forward_raw
ret = self.forward_decode(forward_batch, pp_proxy_tensors=pp_proxy_tensors)
File "/sgl-workspace/sglang/python/sglang/srt/model_executor/model_runner.py", line 1420, in forward_decode
return self.model.forward(
File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py", line 2145, in forward
hidden_states = self.model(input_ids, positions, forward_batch, input_embeds)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py", line 2038, in forward
hidden_states, residual = layer(
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py", line 1888, in forward
hidden_states = self.mlp(hidden_states, forward_batch, can_fuse_mlp_allreduce)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl
return self._call_impl(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl
return forward_call(*args, **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py", line 435, in forward
return self.forward_deepep(hidden_states, forward_batch)
File "/sgl-workspace/sglang/python/sglang/srt/models/deepseek_v2.py", line 578, in forward_deepep
) = self.deepep_dispatcher.dispatch(
File "/sgl-workspace/sglang/python/sglang/srt/two_batch_overlap.py", line 801, in dispatch
return self._execute("dispatch", **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/two_batch_overlap.py", line 798, in _execute
return getattr(self._inners[tbo_subbatch_index or 0], name)(**kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/layers/moe/ep_moe/token_dispatcher.py", line 680, in dispatch
self.dispatch_a(*args, **kwargs)
File "/sgl-workspace/sglang/python/sglang/srt/layers/moe/ep_moe/token_dispatcher.py", line 692, in dispatch_a
inner_state = self._get_impl(forward_batch).dispatch_a(
File "/sgl-workspace/sglang/python/sglang/srt/layers/moe/ep_moe/token_dispatcher.py", line 516, in dispatch_a
hidden_states, masked_m, event, hook = self._dispatch_core(
File "/sgl-workspace/sglang/python/sglang/srt/layers/moe/ep_moe/token_dispatcher.py", line 568, in _dispatch_core
buffer.low_latency_dispatch(
File "/usr/local/lib/python3.10/dist-packages/deep_ep/buffer.py", line 567, in low_latency_dispatch
self.runtime.low_latency_dispatch(x, topk_idx,
RuntimeError: Failed: Assertion error /sgl-workspace/DeepEP/csrc/deep_ep.cpp:1104 'x.size(0) == topk_idx.size(0) and x.size(0) <= num_max_dispatch_tokens_per_rank'