[Bug] 使用LMdeploy在v100推理 Qwen2.5-VL-32B 过一段时间就会卡住

### Checklist

- [ ] 1. I have searched related issues but cannot get the expected help.
- [ ] 2. The bug has not been fixed in the latest version.
- [ ] 3. Please note that if the bug-related issue you submitted lacks corresponding environment info and a minimal reproducible demo, it will be challenging for us to reproduce and resolve the issue, reducing the likelihood of receiving feedback.

### Describe the bug

使用LMdeploy在v100推理 Qwen2.5-VL-32B 过一段时间就会卡住
能正常启动服务，但是跑一会就会在某块卡卡住，其他GPU显示100%利用率，一块卡显示0利用率

### Reproduction

lmdeploy serve api_server /data/phd/hf_models/Qwen2.5-VL-32B-Instruct/ --model-name=Qwen2.5-VL-32B-Instruct --backend turbomind --tp=8 --session-len=32768 --server-port=8080 --max-concurrent-requests=128 --enable-prefix-caching --log-level=DEBUG

### Environment

```Shell
GPU： V100*8
环境安装：uv pip install lmdeploy
```

### Error traceback

```Shell
卡住是最近的日志：
TM][DEBUG] turbomind::core::Tensor turbomind::LlamaV2::postDecodeEmbedding(const turbomind::core::Tensor&, turbomind::core::Buffer)
[TM][DEBUG] void turbomind::LlamaV2::dynamicDecode(turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int)
[TM][DEBUG] void turbomind::LogitsProcessorLayer::Forward(turbomind::core::TensorMap&) [with T = float] start
[TM][DEBUG] void turbomind::LogitsProcessorLayer::Forward(turbomind::core::TensorMap&) [with T = float] stop
[TM][DEBUG] void turbomind::SamplingLayer::Forward(turbomind::core::TensorMap&) [with T = float] start
[TM][DEBUG] void turbomind::SamplingLayer::Forward(turbomind::core::TensorMap&) [with T = float] stop
[TM][DEBUG] void turbomind::StopCriteriaLayer::Forward(turbomind::core::TensorMap&) [with T = float] start
[TM][DEBUG] void turbomind::invokeStopWordsCriterion(const int*, const int*, const int*, bool*, size_t, size_t, int, int, int, cudaStream_t) start
[TM][DEBUG] void turbomind::invokeLengthCriterion(bool*, const int*, int, int, int, cudaStream_t) start
[TM][DEBUG] void turbomind::StopCriteriaLayer::Forward(turbomind::core::TensorMap&) [with T = float] stop
[TM][INFO] [ProcessInferRequests] Request for 99 received.
[TM][INFO] [SeqMgr][Create] ID 99
[TM][WARNING] [ProcessInferRequests] [99] total sequence length (1516 + 31252) exceeds `session_len` (32768), `max_new_tokens` is truncated to 31251
[TM][INFO] [Forward] [0, 7), dc=6, pf=1, sum_q=1522, sum_k=1516, max_q=1516, max_k=1763
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
[TM][DEBUG] void turbomind::LlamaV2::Forward(turbomind::core::Buffer_, turbomind::core::Tensor, turbomind::core::Tensor, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer_, turbomind::core::Buffer_, turbomind::core::Buffer, turbomind::MropeRope*, turbomind::core::Buffer, turbomind::core::Buffer, turbomind::core::Buffer, int, int, const turbomind::Sequence**)
[TM][DEBUG] void turbomind::LlamaV2::updateEmbedding(char*, int, const int*, const turbomind::Sequence**, int, int*, bool*)
```