bf16/float RuntimeError - dtype incompatibility

When training an anima lora, got the following error:

```
[rank3]: Traceback (most recent call last):
[rank3]: File "/workspace/sd-scripts/anima_train_network.py", line 451, in
[rank3]: trainer.train(args)
[rank3]: File "/workspace/sd-scripts/train_network.py", line 1427, in train
[rank3]: loss = self.process_batch(
[rank3]: ^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/anima_train_network.py", line 361, in process_batch
[rank3]: return super().process_batch(
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/train_network.py", line 451, in process_batch
[rank3]: noise_pred, target, timesteps, weighting = self.get_noise_pred_and_target(
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/anima_train_network.py", line 308, in get_noise_pred_and_target
[rank3]: model_pred = anima(
[rank3]: ^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 814, in forward
[rank3]: return model_forward(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 802, in __call__
[rank3]: return convert_to_fp32(self.model_forward(*args, **kwargs))
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
[rank3]: return func(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/library/anima_models.py", line 1375, in forward
[rank3]: return self.forward_mini_train_dit(x, timesteps, context, fps=fps, padding_mask=padding_mask, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/library/anima_models.py", line 1353, in forward_mini_train_dit
[rank3]: x_B_T_H_W_D = block(x_B_T_H_W_D, t_embedding_B_T_D, crossattn_emb, attn_params, use_fp32, **block_kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/library/anima_models.py", line 1020, in forward
[rank3]: return self._forward(
[rank3]: ^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/library/anima_models.py", line 882, in _forward
[rank3]: self.adaln_modulation_self_attn(emb_B_T_D) + adaln_lora_B_T_3D
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/container.py", line 253, in forward
[rank3]: input = module(input)
[rank3]: ^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/networks/lora_anima.py", line 87, in forward
[rank3]: lx = self.lora_down(x)
[rank3]: ^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank3]: return self._call_impl(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank3]: return forward_call(*args, **kwargs)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/linear.py", line 134, in forward
[rank3]: return F.linear(input, self.weight, self.bias)
[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank3]: RuntimeError: expected mat1 and mat2 to have the same dtype, but got: c10::BFloat16 != float
[rank1]: Traceback (most recent call last):
[rank1]: File "/workspace/sd-scripts/anima_train_network.py", line 451, in
[rank1]: trainer.train(args)
[rank1]: File "/workspace/sd-scripts/train_network.py", line 1427, in train
[rank1]: loss = self.process_batch(
[rank1]: ^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/anima_train_network.py", line 361, in process_batch
[rank1]: return super().process_batch(
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/train_network.py", line 451, in process_batch
[rank1]: noise_pred, target, timesteps, weighting = self.get_noise_pred_and_target(
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/anima_train_network.py", line 308, in get_noise_pred_and_target
[rank1]: model_pred = anima(
[rank1]: ^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 814, in forward
[rank1]: return model_forward(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 802, in __call__
[rank1]: return convert_to_fp32(self.model_forward(*args, **kwargs))
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
[rank1]: return func(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/library/anima_models.py", line 1375, in forward
[rank1]: return self.forward_mini_train_dit(x, timesteps, context, fps=fps, padding_mask=padding_mask, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/library/anima_models.py", line 1353, in forward_mini_train_dit
[rank1]: x_B_T_H_W_D = block(x_B_T_H_W_D, t_embedding_B_T_D, crossattn_emb, attn_params, use_fp32, **block_kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/library/anima_models.py", line 1020, in forward
[rank1]: return self._forward(
[rank1]: ^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/library/anima_models.py", line 882, in _forward
[rank1]: self.adaln_modulation_self_attn(emb_B_T_D) + adaln_lora_B_T_3D
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/container.py", line 253, in forward
[rank1]: input = module(input)
[rank1]: ^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/networks/lora_anima.py", line 87, in forward
[rank1]: lx = self.lora_down(x)
[rank1]: ^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank1]: return self._call_impl(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank1]: return forward_call(*args, **kwargs)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/linear.py", line 134, in forward
[rank1]: return F.linear(input, self.weight, self.bias)
[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank1]: RuntimeError: expected mat1 and mat2 to have the same dtype, but got: c10::BFloat16 != float
[rank2]: Traceback (most recent call last):
[rank2]: File "/workspace/sd-scripts/anima_train_network.py", line 451, in
[rank2]: trainer.train(args)
[rank2]: File "/workspace/sd-scripts/train_network.py", line 1427, in train
[rank2]: loss = self.process_batch(
[rank2]: ^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/anima_train_network.py", line 361, in process_batch
[rank2]: return super().process_batch(
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/train_network.py", line 451, in process_batch
[rank2]: noise_pred, target, timesteps, weighting = self.get_noise_pred_and_target(
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/anima_train_network.py", line 308, in get_noise_pred_and_target
[rank2]: model_pred = anima(
[rank2]: ^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 814, in forward
[rank2]: return model_forward(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 802, in __call__
[rank2]: return convert_to_fp32(self.model_forward(*args, **kwargs))
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
[rank2]: return func(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/library/anima_models.py", line 1375, in forward
[rank2]: return self.forward_mini_train_dit(x, timesteps, context, fps=fps, padding_mask=padding_mask, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/library/anima_models.py", line 1353, in forward_mini_train_dit
[rank2]: x_B_T_H_W_D = block(x_B_T_H_W_D, t_embedding_B_T_D, crossattn_emb, attn_params, use_fp32, **block_kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/library/anima_models.py", line 1020, in forward
[rank2]: return self._forward(
[rank2]: ^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/library/anima_models.py", line 882, in _forward
[rank2]: self.adaln_modulation_self_attn(emb_B_T_D) + adaln_lora_B_T_3D
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/container.py", line 253, in forward
[rank2]: input = module(input)
[rank2]: ^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/networks/lora_anima.py", line 87, in forward
[rank2]: lx = self.lora_down(x)
[rank2]: ^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank2]: return self._call_impl(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank2]: return forward_call(*args, **kwargs)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/linear.py", line 134, in forward
[rank2]: return F.linear(input, self.weight, self.bias)
[rank2]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank2]: RuntimeError: expected mat1 and mat2 to have the same dtype, but got: c10::BFloat16 != float
[rank0]: Traceback (most recent call last):
[rank0]: File "/workspace/sd-scripts/anima_train_network.py", line 451, in
[rank0]: trainer.train(args)
[rank0]: File "/workspace/sd-scripts/train_network.py", line 1427, in train
[rank0]: loss = self.process_batch(
[rank0]: ^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/anima_train_network.py", line 361, in process_batch
[rank0]: return super().process_batch(
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/train_network.py", line 451, in process_batch
[rank0]: noise_pred, target, timesteps, weighting = self.get_noise_pred_and_target(
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/anima_train_network.py", line 308, in get_noise_pred_and_target
[rank0]: model_pred = anima(
[rank0]: ^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 814, in forward
[rank0]: return model_forward(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/utils/operations.py", line 802, in __call__
[rank0]: return convert_to_fp32(self.model_forward(*args, **kwargs))
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/amp/autocast_mode.py", line 44, in decorate_autocast
[rank0]: return func(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/library/anima_models.py", line 1375, in forward
[rank0]: return self.forward_mini_train_dit(x, timesteps, context, fps=fps, padding_mask=padding_mask, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/library/anima_models.py", line 1353, in forward_mini_train_dit
[rank0]: x_B_T_H_W_D = block(x_B_T_H_W_D, t_embedding_B_T_D, crossattn_emb, attn_params, use_fp32, **block_kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/library/anima_models.py", line 1020, in forward
[rank0]: return self._forward(
[rank0]: ^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/library/anima_models.py", line 882, in _forward
[rank0]: self.adaln_modulation_self_attn(emb_B_T_D) + adaln_lora_B_T_3D
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/container.py", line 253, in forward
[rank0]: input = module(input)
[rank0]: ^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/networks/lora_anima.py", line 87, in forward
[rank0]: lx = self.lora_down(x)
[rank0]: ^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1778, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/module.py", line 1789, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/nn/modules/linear.py", line 134, in forward
[rank0]: return F.linear(input, self.weight, self.bias)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: RuntimeError: expected mat1 and mat2 to have the same dtype, but got: c10::BFloat16 != float
steps: 0%| | 0/4860 [00:02
sys.exit(main())
^^^^^^
File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
args.func(args)
File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/commands/launch.py", line 1204, in launch_command
multi_gpu_launcher(args)
File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/accelerate/commands/launch.py", line 825, in multi_gpu_launcher
distrib_run.run(args)
File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/distributed/run.py", line 1007, in run
elastic_launch(
File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 191, in __call__
return launch_agent(self._config, self._entrypoint, list(args))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/workspace/sd-scripts/venv/lib/python3.12/site-packages/torch/distributed/launcher/api.py", line 371, in launch_agent
raise ChildFailedError(
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
============================================================
anima_train_network.py FAILED
------------------------------------------------------------
Failures:
[1]:
time : 2026-05-29_04:55:36
host : aa9273e2025b
rank : 0 (local_rank: 0)
exitcode : -15 (pid: 98061) (SIGTERM)
error_file:
traceback : Signal 15 (SIGTERM) received by PID 98061
[2]:
time : 2026-05-29_04:55:36
host : aa9273e2025b
rank : 2 (local_rank: 2)
exitcode : -15 (pid: 98063) (SIGTERM)
error_file:
traceback : Signal 15 (SIGTERM) received by PID 98063
[3]:
time : 2026-05-29_04:55:36
host : aa9273e2025b
rank : 3 (local_rank: 3)
exitcode : 1 (pid: 98064)
error_file:
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
------------------------------------------------------------
Root Cause (first observed failure):
[0]:
time : 2026-05-29_04:55:35
host : aa9273e2025b
rank : 1 (local_rank: 1)
exitcode : 1 (pid: 98062)
error_file:
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
```

My training command:

accelerate launch --num_cpu_threads_per_process 2 --num_processes 4 --num_machines 1 --multi_gpu --gpu_ids 0,1,2,3 anima_train_network.py --pretrained_model_name_or_path dataset/anima-base-v1.0.safetensors --qwen3 dataset/qwen_3_06b_base.safetensors --vae dataset/qwen_image_vae.safetensors --dataset_config dataset/output_config.toml --output_dir dataset/models --output_name bleach_71_anima --save_precision bf16 --save_model_as safetensors --save_every_n_epochs 1 --network_module networks.lora_anima --network_dim 32 --network_alpha 32 --network_train_unet_only --learning_rate 4e-05 --optimizer_type optimi.AdamW --lr_scheduler cosine --loss_type l2 --train_batch_size 4 --lr_warmup_steps 0.1 --gradient_accumulation 1 --max_grad_norm 0.5 --training_comment "unet only" --timestep_sampling sigmoid --discrete_flow_shift 1.0 --sigmoid_scale 1.3 --max_train_epochs 10 --max_data_loader_n_workers 16 --persistent_data_loader_workers --mixed_precision bf16 --qwen3_max_token_length 512 --t5_max_token_length 512 --resolution 1024 --cache_latents --cache_latents_to_disk --logging_dir dataset/logs --log_tracker_name anima --log_with tensorboard --caption_separator ". " --no_metadata --enable_bucket --min_bucket_reso 256 --max_bucket_reso 2048 --bucket_reso_steps 64 --bucket_no_upscale --network_args train_llm_adapter=False "include_patterns=['.*_modulation.*']" --optimizer_args weight_decay=0 "betas=(0.9,0.99)" eps=1e-06

Passing mixed_precision 16 to accelerate led to the same issue. Using regular AdamW instead of optimi.AdamW also had the same issue. The only thing that seems to fix things is running in --full_bf16 . Even so, when running in full_bf16, I eventually get NaNs for loss a thousand or so steps into training.

Start at networks/lora_anima.py line 87, then trace the bf16 input from library/anima_models.py lines 882 and 1353-1375. Reproduce the Anima LoRA training failure and inspect how the LoRA linear layers and model autocasting choose dtypes. Done means the reported bf16/float RuntimeError no longer occurs during training.

Written by the indexing model from the issue text.