mindspore-ai / mindspore-ai/hyper-parallel

111

Open
#656 0 comments 0 reactions 0 assignees View on GitHub

Nobody has claimed this yet.

Dominant language
Python
Stars
53
Forks
63
Avg merge
23h 45m
Merged PRs (30d)
63

Description

checkpoint:
  async_save: false
  enable_save: false
  load_balanced: false
  load_path: ''
  load_worker_number: 1
  no_load_optim: true
  no_save_optim: false
  prefix: custom
  remove_redundancy: false
  save_interleaved_steps: 100
  save_max: 1
  save_path: ''
training:
  deterministic: false
  global_batch_size: 1024
  local_batch_size: 1
  max_norm: 1.0
  seed: 42
  steps: 8000000000
optimizer:
  adamw_betas:
  - 0.9
  - 0.95
  adamw_eps: 1.0e-08
  type: Muon
  use_fused_adamw: true
  weight_decay: 0.1
  adamw_include:
  - '*word_embeddings*'
  - '*output_layer*'
  matched_adamw_rms: 0.2
  momentum: 0.95
  nesterov: true
  ns_coefficients: [[[3.4445, -4.7750, 2.0315], 8], [[2.0, -1.5, 0.5], 2]]
  ns_steps: 8
  comm_strategy: allgather_deredundency
  use_fused_adamw: True
  phase0_batch_memory_gb: 1     # 动量预处理 → 主要影响 reserved 高水位
  ns_batch_memory_gb: 1         # Newton-Schulz → 主要影响 allocated 峰值
  phase4_batch_memory_gb: 4
  qk_clip_enabled: true
  qk_clip_threshold: 100.0
lr_scheduler:
  type: ConstantWarmUpLR
  learning_rate: 0.0001
  warmup_ratio: 0
parallelism:
  context_parallel: 1
  context_parallel_method: colossal
  data_parallel_shard: -1
  data_parallel_shard_strategy: optim_grads_params
  enable_loss_parallel: true
  enable_mc2: true
  expert_parallel: 64
  sharded_accumulated_grad: false
  expert_parallel_async_d2h: true
  expert_parallel_use_safe_tokens: false
  moe_token_dispatcher_type: alltoall
  pipeline_parallel: 8
  pipeline_parallel_interleave_num: 2
  pipeline_parallel_layers_per_stage:
  - 0-2,31-34
  - 3-6,35-37
  - 7-10,38-41
  - 11-14,42-45
  - 15-18,46-49
  - 19-22,50-53
  - 23-26,54-58
  - 27-30,59-60
  pipeline_parallel_overlap_b_f: true
  pipeline_parallel_overlap_p2p: true
  pipeline_parallel_p2p_transport: "edge" #"multi_stream"
  pipeline_parallel_schedule: 1f1b
  sequence_parallel: true
  tensor_parallel: 4
model:
  add_bias_linear: false
  apply_rope_fusion: true
  architectures: DeepseekV4ForCausalLM
  attention_dropout: 0.0
  chunk_loss_num: 4
  compute_dtype: bfloat16
  hidden_act: "fusedswiglu" #silu
  swiglu_limit: 10.0
  hidden_dropout: 0.0
  hidden_size: 7168
  initializer_range: 0.02
  kv_lora_rank: 512
  layernorm_compute_dtype: float32
  max_position_embeddings: 4096
  model_type: deepseek_v4
  moe_aux_loss_coeff: 0.0006
  moe_grouped_gemm: true
  moe_intermediate_size: 2048
  moe_permute_fusion: true
  moe_router_bias_update_rate: 0.002
  moe_router_enable_expert_bias: true
  moe_router_force_expert_balance: true
  moe_router_load_balancing_type: seq_aux_loss
  moe_shared_expert_intermediate_size: 2048
  mtp_loss_scaling_factor: 0.3
  multi_latent_attention: true
  n_routed_experts: 384
  n_shared_experts: 1
  num_attention_heads: 128
  num_experts_per_tok: 8
  num_hidden_layers: 61
  num_nextn_predict_layers: 1
  params_dtype: bfloat16
  position_embedding_type: yarn
  q_lora_rank: 1536
  qk_layernorm: true
  qk_nope_head_dim: 128
  qk_rope_head_dim: 64
  rms_norm_eps: 1.0e-06
  rope_theta: 10000
  rotary_dtype: float32
  routed_scaling_factor: 2.5
  scoring_func: sigmoid
  seq_length: 4096
  softmax_compute_dtype: float32
  use_flash_attention: true
  v_head_dim: 128
  vocab_size: 131072
  enable_hc_head: false
  enable_hyper_connections: true
  use_fused_mhc: True
  experimental_attention_variant: mla
  hc_eps: 1.0e-06
  hc_mult: 4
  hc_sinkhorn_iters: 20
  mla_qkv_concat: true
  moe_router_pre_softmax: true
  moe_router_score_function: sigmoid
  moe_router_topk_scaling_factor: 2.5
  moe_token_dispatcher_type: alltoall
  moe_token_drop_policy: probs
  mtp_loss_factor: 0.3
  normalization: RMSNorm
  num_hash_layers: 3
  qk_clip: true
recompute:
  mode: select #select
  full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17,  19-21, 23-25, 27-28,  31-32, 35-36, 38-40, 42-44, 46-47,  50-51, 54-55]
  #full_recompute_layer: [0-2, 3-4, 7-8, 11-12, 15-16,  19-20, 23-24, 27-28,  31-32, 35-36, 38-41, 42-44, 46-47,  50-51, 54-55]
  #full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17,  19-21, 23-25, 27-29,  31-33, 35-36, 38-40, 42-44, 46-48,  50-52, 54-56, 59]
  #full_recompute_layer: [0-61]
  #exclude_op: ['AllToAll']

  select_module:
    mlp.experts.activation_func: ["0-61"]
    mlp.shared_experts.activation_func: ["0-61"]
    transformer_layer.mlp.experts.activation_func: ["62"]
    transformer_layer.mlp.shared_experts.activation_func: ["62"]
    #recompute:
    #  mode: full
  #full_recompute_layer: [0-3, 4-7, 8-11, 12-15, 16-19,  20-23, 24-27, 28-31,  32-35, 36-39, 40-43, 44-47, 48-51,  52-54, 56-58, 60]
  #full_recompute_layer: [0-2, 3-5, 7-9, 11-13, 15-17,  19-21, 23-25, 27-28,  31-32, 35-36, 38-40, 42-44, 46-47,  50-51, 54-55]
  #full_recompute_layer: [0-2, 4-6, 8-10, 12-14, 16-18,  20-22, 24-26, 28-30,  32-34, 35-37, 38-40, 42-44, 46-47,  50-52, 54]
  #full_recompute_layer: [0-61]
  #full_recompute_layer: #[0-2,4-6,8-10,12-14,16-18,20-22,24-26,28-30,32-34,36-38,40-42,44-46,48-50,52-54,56-57,60-61] #[0-1,4-5,8-9,12-13,16-17,20-21,24-25,28-29,32-33,36-37,40-41,44-45,48-49,52-53,556-57,60] #[0-1,4-5,8-9,12-13,16-17,20-21,24-25,28] #[0-31]
  #exclude_op: ['AllToAll']
train_dataset:
  dataloader:
    column_names:
    - input_ids
    - labels
    - loss_mask
    - position_ids
    type: BlendedMegatronDatasetDataLoader
    config:
      create_attention_mask: false
      create_compressed_eod_mask: false
      data_path:
      - '1'
      - /hpfs/huawei/public/deepseek4_flash/datasets/mmap_deepseekv4_datasets_text_document
      eod: 1
      eod_mask_loss: false
      eod_pad_length: 128
      pad: -1
      reset_attention_mask: false
      reset_position_ids: false
      seed: 1234
      seq_length: 4096
      split: 1, 0, 0
    datasets_type: GPTDataset
    sizes:
    - 512000
    - 0
    - 0
profiler:
  enable_profiling: True #True
  end_step: 5
  output_path: /hpfs/huawei/0727lc/prof_lc0728_1513_level1
  profile_cpu: False
  profile_memory: False
  profiler_level: 1
  profiler_rank: [0, 128, 256, 384, 512, 640, 768, 896] 
  #profiler_rank: [768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800] 
  #profiler_rank: [768,774,784,785,787,788,792,800,808,816,824,830] 
  start_step: 5
  with_stack: False

schema_version: 1
source: gitcode
gitcode_repo: mindspore/hyper-parallel
gitcode_issue: 308
source_url: https://gitcode.com/mindspore/hyper-parallel/issues/308

Contributor guide

No contributing guide indexed for this repository

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Research direction

The report contains only a large YAML training configuration and no requested change, failure, or named repository file. Start by clarifying the intended problem and the relevant configuration section; completion criteria and a test or reproduction are not provided.

Written by the indexing model from the issue text.

Assessment

Tech stack
python
Domain
distributed-systems, machine-learning
Issue type
Bug
Difficulty
5/5
Estimated time
Over a week
Activity status
Active
Clarity
Needs clarification
Newbie friendliness
10/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.