Error when saving checkpoints with the Megatron backend

Checklist / 检查清单

• I have searched existing issues, and this is a new bug report. / 我已经搜索过现有的 issues，确认这是一个新的 bug report。

Bug Description / Bug 描述

Using the v4.0.1 image and code (without any modifications), I trained Qwen3.5-27B with the Megatron backend.

An error occurs when saving the second checkpoint (when saving optimizer state and RNG state).

!!!!!!! Segfault encountered !!!!!!!
  File "./signal/../sysdeps/unix/sysv/linux/x86_64/libc_sigaction.c", line 0, in 0x00007fed7cfcb51f
  File "<unknown>", line 0, in crc32_16bytes(void const*, unsigned long, unsigned int)
  File "<unknown>", line 0, in mz_crc32
  File "<unknown>", line 0, in mz_zip_writer_add_mem_ex_v2
  File "<unknown>", line 0, in caffe2::serialize::PyTorchStreamWriter::writeRecord(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, void const*, unsigned long, bool)
  File "<unknown>", line 0, in pybind11::cpp_function::initialize<torch::jit::initJITBindings(_object*)::{lambda(caffe2::serialize::PyTorchStreamWriter&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::Storage const&, unsigned long)#1}, void, caffe2::serialize::PyTorchStreamWriter&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::Storage const&, unsigned long, pybind11::name, pybind11::is_method, pybind11::sibling>(torch::jit::initJITBindings(_object*)::{lambda(caffe2::serialize::PyTorchStreamWriter&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::Storage const&, unsigned long)#1}&&, void (*)(caffe2::serialize::PyTorchStreamWriter&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, c10::Storage const&, unsigned long), pybind11::name const&, pybind11::is_method const&, pybind11::sibling const&)::{lambda(pybind11::detail::function_call&)#1}::_FUN(pybind11::detail::function_call&)
  File "<unknown>", line 0, in pybind11::cpp_function::dispatcher(_object*, _object*, _object*)
  File "Objects/methodobject.c", line 542, in cfunction_call
  File "Objects/call.c", line 214, in _PyObject_MakeTpCall
  File "Python/ceval.c", line 4769, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "Objects/call.c", line 257, in _PyVectorcall_Call
  File "Objects/call.c", line 355, in PyObject_Call
  File "Python/ceval.c", line 7349, in do_call_core
  File "Python/ceval.c", line 5376, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "Objects/call.c", line 257, in _PyVectorcall_Call
  File "Objects/call.c", line 355, in PyObject_Call
  File "Python/ceval.c", line 7349, in do_call_core
  File "Python/ceval.c", line 5376, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "./Include/internal/pycore_call.h", line 92, in _PyObject_VectorcallTstate
  File "./Modules/_functoolsmodule.c", line 233, in partial_vectorcall
  File "Objects/call.c", line 257, in _PyVectorcall_Call
  File "Objects/call.c", line 355, in PyObject_Call
  File "Python/ceval.c", line 7349, in do_call_core
  File "Python/ceval.c", line 5376, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "Objects/call.c", line 393, in _PyFunction_Vectorcall
  File "Objects/call.c", line 141, in _PyObject_FastCallDictTstate
  File "Objects/call.c", line 482, in _PyObject_Call_Prepend
  File "Objects/typeobject.c", line 7855, in slot_tp_init
  File "Objects/typeobject.c", line 1103, in type_call
  File "Objects/call.c", line 214, in _PyObject_MakeTpCall
  File "Python/ceval.c", line 4769, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "Objects/call.c", line 355, in PyObject_Call
  File "Python/ceval.c", line 7349, in do_call_core
  File "Python/ceval.c", line 5376, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "./Include/internal/pycore_call.h", line 92, in _PyObject_VectorcallTstate
  File "./Modules/_functoolsmodule.c", line 260, in partial_vectorcall
  File "Objects/call.c", line 355, in PyObject_Call
  File "Python/ceval.c", line 7349, in do_call_core
  File "Python/ceval.c", line 5376, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "Objects/call.c", line 257, in _PyVectorcall_Call
  File "Objects/call.c", line 355, in PyObject_Call
  File "Python/ceval.c", line 7349, in do_call_core
  File "Python/ceval.c", line 5376, in _PyEval_EvalFrameDefault
  File "./Include/internal/pycore_ceval.h", line 73, in _PyEval_EvalFrame
  File "Python/ceval.c", line 6434, in _PyEval_Vector
  File "Python/ceval.c", line 1148, in PyEval_EvalCode
  File "Python/pythonrun.c", line 1741, in run_eval_code_obj
  File "Python/pythonrun.c", line 1762, in run_mod
  File "Python/pythonrun.c", line 1657, in pyrun_file
  File "Python/pythonrun.c", line 440, in _PyRun_SimpleFileObject
  File "Python/pythonrun.c", line 79, in _PyRun_AnyFileObject
  File "Modules/main.c", line 360, in pymain_run_file_obj
  File "Modules/main.c", line 379, in pymain_run_file
  File "Modules/main.c", line 605, in pymain_run_python
  File "Modules/main.c", line 684, in Py_RunMain
  File "Modules/main.c", line 738, in Py_BytesMain
  File "<unknown>", line 0, in _start
  File "<unknown>", line 0, in 0xffffffffffffffff

If saving the optimizer state and RNG state is disabled, the error no longer occurs.

How to Reproduce / 如何复现

PYTORCH_CUDA_ALLOC_CONF='expandable_segments:True' \
NNODES=2 \
NODE_RANK=${RANK} \
MASTER_ADDR=${MASTER_ADDR} \
MASTER_PORT=6000 \
NPROC_PER_NODE=8 \
python -m swift.cli._megatron.main sft \
    --seed 42 \
    --ddp_timeout 18000000 \
    --report_to wandb \
    --wandb_project test \
    --wandb_exp_name test \
    --logging_steps 1 \
    --model ... \
    --output_dir ... \
    --bf16 true \
    --check_model false \
    --save_safetensors true \
    --max_shard_size 5GB \
    --cached_dataset ... \
    --split_dataset_ratio 0 \
    --data_seed 42 \
    --dataset_shuffle true \
    --truncation_strategy delete \
    --padding_side right \
    --train_dataloader_shuffle true \
    --add_non_thinking_prefix true \
    --dataset_num_proc 8 \
    --dataloader_num_workers 8 \
    --dataloader_prefetch_factor 2 \
    --max_length 6144 \
    --packing false \
    --packing_length 16384 \
    --packing_num_proc 80 \
    --padding_free true \
    --load_from_cache_file true \
    --tuner_type full \
    --loss_scale default \
    --micro_batch_size 1 \
    --global_batch_size 32 \
    --lr $lr \
    --lr_decay_style cosine \
    --lr_warmup_fraction 0.05 \
    --min_lr 0 \
    --weight_decay 0.1 \
    --clip_grad 1.0 \
    --adam_beta1 0.9 \
    --adam_beta2 0.95 \
    --adam_eps 1e-8 \
    --sgd_momentum 0.9 \
    --save_strategy steps \
    --eval_steps 1000000 \
    --save_steps $save_steps \
    --no_save_optim true \
    --no_save_rng true \
    --tensor_model_parallel_size 4 \
    --pipeline_model_parallel_size 2 \
    --context_parallel_size 1 \
    --recompute_granularity full \
    --recompute_method uniform \
    --recompute_num_layers 1 \
    --num_train_epochs 1 \
    --group_by_length false \
    --finetune true \
    --freeze_llm false \
    --freeze_vit true \
    --freeze_aligner true \
    --sequence_parallel true \
    --attention_backend flash

Additional Information / 补充信息

No response

Start at the swift.cli._megatron.main entry point and trace the checkpoint path used when optimizer and RNG state are saved. Reproduce the two-node command with those options enabled, then compare it with the no-save-optimizer/no-save-rng case. Done means the second checkpoint completes without a segmentation fault while both states remain enabled.