deepspeed zero3 model_update error

```yaml
defaults:
- ../config/deepspeed_zero@_here_
- ../config/deepspeed_zero2@_here_
- ../config/deepspeed_zero3@_here_
- ../config/deepspeed_zero3_cpuoffload@_here_

hydra:
run:
dir: .
output_subdir: null

exp_name: "qwen2.5-7B-grpo-zero3-vllm-codesandbox"
seed: 42
logging_dir: ./output/logs
output_dir: ./output

system_envs:
USE_MODELSCOPE: '1'

checkpoint_config:
type: file_system
output_dir: ./output/checkpoint/${exp_name}

track_with: tensorboard
tracker_kwargs:
log_dir: ./output/tensorboard/${exp_name}

num_gpus_per_node: 8

max_steps: 500
save_steps: 100
logging_steps: 1
eval_steps: 10
resume_from_checkpoint: false

# --------------------------
# GRPO 相关
rollout_batch_size: 64
adv_estimator: "grpo"
num_return_sequences_in_group: 4
prompt_length: 2048
response_length: 2048

ppo_epochs: 1
use_kl_loss: true
kl_loss_coef: 0.001
loss_agg_mode: "seq-mean-token-sum"

# PPO 相关
whiten_advantages: true
advantage_clip: 2.0
dual_clip_loss: true
reward_clip: 10
reward_norm: null
reward_shift: false
reward_scale: false
add_token_level_kl: false

# --------------------------
# 模型与数据
pretrain: Seed-Coder-8B-Instruct
reward_pretrain: ${pretrain}

validation:
data_args:
template: native
file_name:
- data/code_KodCode_data.jsonl
generating_args:
max_new_tokens: ${response_length}
top_p: 0.6
top_k: 50
num_beams: 1
temperature: 0.6
num_return_sequences: 1

actor_train:
model_args:
attn_implementation: fa2
disable_gradient_checkpointing: false
dtype: bf16
model_type: ~
training_args:
learning_rate: 1.0e-6
weight_decay: 0
per_device_train_batch_size: 1
gradient_accumulation_steps: 16
warmup_steps: 20
num_train_epochs: 50
data_args:
template: native
file_name:
- data/code_KodCode_data.jsonl
dataset_dir: data
preprocessing_num_workers: 8
domain_interleave_probs:
code_sandbox: 1.0
strategy_args:
strategy_name: deepspeed_train
strategy_config: ${deepspeed_zero3_cpuoffload}
device_mapping: "[0,1,2,3,4,5]"
infer_batch_size: 1

actor_infer:
model_args:
disable_gradient_checkpointing: true
dtype: bf16
generating_args:
max_new_tokens: ${response_length}
top_p: 0.95
top_k: 50
num_beams: 1
temperature: 0.8
num_return_sequences: ${num_return_sequences_in_group}
data_args:
template: qwen2_5
strategy_args:
strategy_name: vllm
strategy_config:
tensor_parallel_size: 2
gpu_memory_utilization: 0.8
block_size: 16
max_model_len: 4096
device_mapping: "[6,7]"
infer_batch_size: 1

reference:
model_args:
disable_gradient_checkpointing: true
dtype: bf16
data_args:
template: qwen2_5
strategy_args:
strategy_name: hf_infer
strategy_config: null
device_mapping: "[6,7]"
infer_batch_size: 1

rewards:
code_sandbox:
use_local: true
worker_cls: roll.pipeline.rlvr.rewards.code_sandbox_reward_worker.CodeSandboxRewardWorker
tag_included: [assert, input, livecodebench_random100, default, code, code_sandbox, KodCode]
model_args:
model_name_or_path: ${reward_pretrain}
data_args:
template: native
world_size: 8
infer_batch_size: 1
query_filter_config:
type: std_filter
filter_args:
std_threshold: 0
```

设置train "[0,1,2,3,4,5]", infer [6,7]"会导致NCLL 错误。 如果都使用list(range(0,8))则可以正常运行。所以是不是代码中参数同步或者zero gather发生了错误？

Start with the provided configuration, especially actor_train.strategy_args and actor_infer.strategy_args, and reproduce the NCCL error using device_mapping [0,1,2,3,4,5] for training and [6,7] for inference. Compare this with list(range(0,8)) and determine whether parameter synchronization or zero gathering fails; done means the cause and affected path are identified.

Written by the indexing model from the issue text.