Error occurs when saving the model during megtron training

An error occurred when saving at step 100

The error is as follows:
```
(ActorWorker(reference-2) pid=12088, ip=172.19.255.125) Elapsed time: 0.4288 seconds
(ActorWorker(reference-0) pid=11942, ip=172.19.255.125) Elapsed time: 59.2630 seconds [repeated 8x across cluster]
Traceback (most recent call last):
File "/ROLL/examples/start_rlvr_pipeline.py", line 37, in
main()
File "/ROLL/examples/start_rlvr_pipeline.py", line 33, in main
pipeline.run()
File "/opt/conda/envs/verl_hiyouga/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
return func(*args, **kwargs)
File "/ROLL/roll/pipeline/rlvr/rlvr_pipeline.py", line 528, in run
self.do_checkpoint(global_step=global_step)
File "/ROLL/roll/pipeline/base_pipeline.py", line 82, in do_checkpoint
ckpt_metrics = DataProto.materialize_concat(data_refs=ckpt_metrics_refs)
File "/ROLL/roll/distributed/scheduler/protocol.py", line 734, in materialize_concat
data = ray.get(data_refs, timeout=timeout)
File "/opt/conda/envs/verl_hiyouga/lib/python3.10/site-packages/ray/_private/auto_init_hook.py", line 21, in auto_init_wrapper
return fn(*args, **kwargs)
File "/opt/conda/envs/verl_hiyouga/lib/python3.10/site-packages/ray/_private/client_mode_hook.py", line 103, in wrapper
return func(*args, **kwargs)
File "/opt/conda/envs/verl_hiyouga/lib/python3.10/site-packages/ray/_private/worker.py", line 2771, in get
values, debugger_breakpoint = worker.get_objects(object_refs, timeout=timeout)
File "/opt/conda/envs/verl_hiyouga/lib/python3.10/site-packages/ray/_private/worker.py", line 893, in get_objects
] = self.core_worker.get_objects(
File "python/ray/_raylet.pyx", line 3189, in ray._raylet.CoreWorker.get_objects
File "python/ray/includes/common.pxi", line 85, in ray._raylet.check_status
ray.exceptions.GetTimeoutError: Get timed out: some object(s) not ready.
(ActorWorker(reference-6) pid=12093, ip=172.19.255.125) Elapsed time: 61.3996 seconds [repeated 23x across cluster]
```

The example is as follows:：
```
hydra:
run:
dir: .
output_subdir: null

exp_name: "qwen3-code-moe"
seed: 42
logging_dir: ./output/logs
output_dir: ./output
system_envs:
USE_MODELSCOPE: '1'

checkpoint_config:
type: file_system
output_dir: /ROLL/outputcheckpoint

track_with: tensorboard
tracker_kwargs:
log_dir: ./rl_examples/llm/tensorboard/roll_exp/rlvr

num_gpus_per_node: 8

max_steps: 500
save_steps: 100
logging_steps: 1
eval_steps: 10
resume_from_checkpoint: false

rollout_batch_size: 64 # prompt
prompt_length: 2048
response_length: 4096

num_return_sequences_in_group: 8
ppo_epochs: 1
adv_estimator: "reinforce"

# clip
value_clip: 0.5
reward_clip: 10
advantage_clip: 2.0
dual_clip_loss: true

# normalize
reward_norm: null
reward_shift: false
reward_scale: false

# data mask
max_len_mask: true
difficulty_mask: true
difficulty_low_threshold: 0.1
difficulty_high_threshold: 0.95
error_max_len_clip: false

# data weight
difficulty_loss_weight: false
length_loss_weight: false

# reward
add_token_level_kl: false

# advantage
whiten_advantages: true

# dynamic sampling scheduler
# use_additional_prompts: true
# max_running_requests: 256
# is_num_return_sequences_expand: false

pretrain: /Qwen/Qwen3-Coder-30B-A3B-Instruct
reward_pretrain: Qwen/Qwen3-Coder-30B-A3B-Instruct

validation:
data_args:
template: qwen3
file_name:
- data/code_KodCode_data.jsonl
generating_args:
max_new_tokens: ${response_length}
top_p: 0.6
top_k: 50
num_beams: 1
temperature: 0.7
num_return_sequences: 1

actor_train:
model_args:
disable_gradient_checkpointing: false
dtype: bf16
model_type: ~
training_args:
learning_rate: 1.0e-6
weight_decay: 0
per_device_train_batch_size: 1
gradient_accumulation_steps: 8
warmup_steps: 20
num_train_epochs: 50
data_args:
template: qwen3
file_name:
- data/code_KodCode_data.jsonl
domain_interleave_probs:
code_sandbox: 1.0
dataset_dir: data
messages: messages
interleave_probs: "1.0"
preprocessing_num_workers: 16
strategy_args:
strategy_name: megatron_train
strategy_config:
tensor_model_parallel_size: 4
pipeline_model_parallel_size: 4
expert_model_parallel_size: 2
use_distributed_optimizer: true
recompute_granularity: full
device_mapping: list(range(0,32))
infer_batch_size: 4

actor_infer:
model_args:
disable_gradient_checkpointing: true
dtype: bf16
generating_args:
max_new_tokens: ${response_length}
top_p: 0.99
top_k: 100
num_beams: 1
temperature: 0.99
num_return_sequences: ${num_return_sequences_in_group}
data_args:
template: qwen3
strategy_args:
strategy_name: vllm
strategy_config:
tensor_parallel_size: 8
gpu_memory_utilization: 0.8
block_size: 16
max_model_len: 8000
device_mapping: list(range(32,48))
infer_batch_size: 1

reference:
model_args:
disable_gradient_checkpointing: true
dtype: bf16
model_type: ~
data_args:
template: qwen3
strategy_args:
strategy_name: megatron_infer
strategy_config:
tensor_model_parallel_size: 4
pipeline_model_parallel_size: 2
expert_model_parallel_size: 2
device_mapping: list(range(48,64))
infer_batch_size: 8

rewards:
code_sandbox:
use_local: true
worker_cls: roll.pipeline.rlvr.rewards.code_sandbox_reward_worker.CodeSandboxRewardWorker
tag_included: [KodCode]
model_args:
model_name_or_path: ${reward_pretrain}
data_args:
template: qwen3
world_size: 8
infer_batch_size: 1
math_rule:
worker_cls: roll.pipeline.rlvr.rewards.math_rule_reward_worker.MathRuleRewardWorker
model_args:
model_name_or_path: ${reward_pretrain}
data_args:
template: qwen3
tag_included: [deepmath_103k, aime]
world_size: 8
infer_batch_size: 1
```