kohya-ss / kohya-ss/sd-scripts
Flux training, no matter the settings I get: returned non-zero exit status 3221225477
- Dominant language
- Python
- Stars
- 7.2k
- Forks
- 1.2k
- Avg merge
- 11m
- Merged PRs (30d)
- 2
Description
I'm trying to run a lora training with flux.. I'm following this guide: https://civitai.com/articles/9487
And the preset parameters from this configure file look like this:
{
"LoRA_type": "Flux1",
"LyCORIS_preset": "full",
"adaptive_noise_scale": 0,
"additional_parameters": "",
"ae": "S:\\path\\to\\kohya_ss\\models\\ae.safetensors",
"apply_t5_attn_mask": true,
"async_upload": false,
"block_alphas": "",
"block_dims": "",
"block_lr_zero_threshold": "",
"blocks_to_swap": 23,
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"bypass_mode": false,
"cache_latents": true,
"cache_latents_to_disk": true,
"caption_dropout_every_n_epochs": 0,
"caption_dropout_rate": 0,
"caption_extension": ".txt",
"clip_l": "S:\\path\\to\\kohya_ss\\models\\clip_l.safetensors",
"clip_skip": 0,
"color_aug": false,
"constrain": 0,
"conv_alpha": 1,
"conv_block_alphas": "",
"conv_block_dims": "",
"conv_dim": 1,
"cpu_offload_checkpointing": false,
"dataset_config": "",
"debiased_estimation_loss": false,
"decompose_both": false,
"dim_from_weights": false,
"discrete_flow_shift": 3.1582,
"dora_wd": false,
"double_blocks_to_swap": 0,
"down_lr_weight": "",
"dynamo_backend": "no",
"dynamo_mode": "default",
"dynamo_use_dynamic": false,
"dynamo_use_fullgraph": false,
"enable_all_linear": false,
"enable_bucket": true,
"epoch": 200,
"extra_accelerate_launch_args": "",
"factor": -1,
"flip_aug": true,
"flux1_cache_text_encoder_outputs": true,
"flux1_cache_text_encoder_outputs_to_disk": true,
"flux1_checkbox": true,
"fp8_base": true,
"fp8_base_unet": false,
"full_bf16": true,
"full_fp16": false,
"gpu_ids": "1",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"guidance_scale": 1,
"highvram": true,
"huber_c": 0.1,
"huber_schedule": "snr",
"huggingface_path_in_repo": "",
"huggingface_repo_id": "",
"huggingface_repo_type": "",
"huggingface_repo_visibility": "",
"huggingface_token": "",
"img_attn_dim": "",
"img_mlp_dim": "",
"img_mod_dim": "",
"in_dims": "",
"ip_noise_gamma": 0,
"ip_noise_gamma_random_strength": false,
"keep_tokens": 0,
"learning_rate": 0.0006,
"log_config": false,
"log_tracker_config": "",
"log_tracker_name": "",
"log_with": "",
"logging_dir": "W:/path/to/training/log",
"loraplus_lr_ratio": 0,
"loraplus_text_encoder_lr_ratio": 0,
"loraplus_unet_lr_ratio": 0,
"loss_type": "l2",
"lowvram": false,
"lr_scheduler": "constant",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": 1,
"lr_scheduler_power": 1,
"lr_scheduler_type": "",
"lr_warmup": 0,
"lr_warmup_steps": 0,
"main_process_port": 0,
"masked_loss": false,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": 0,
"max_grad_norm": 0,
"max_resolution": "512,512",
"max_timestep": 1000,
"max_token_length": 225,
"max_train_epochs": 0,
"max_train_steps": 1200,
"mem_eff_attn": false,
"mem_eff_save": false,
"metadata_author": "Metadata author",
"metadata_description": "Metadata description",
"metadata_license": "Metadata license",
"metadata_tags": "Metadata tags",
"metadata_title": "Metadata title",
"mid_lr_weight": "",
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "custom",
"model_prediction_type": "raw",
"module_dropout": 0,
"multi_gpu": false,
"multires_noise_discount": 0.3,
"multires_noise_iterations": 0,
"network_alpha": 32,
"network_dim": 32,
"network_dropout": 0,
"network_weights": "",
"noise_offset": 0.03,
"noise_offset_random_strength": false,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 1,
"num_machines": 1,
"num_processes": 1,
"optimizer": "Adafactor",
"optimizer_args": "scale_parameter=False relative_step=False warmup_init=False weight_decay=0.01",
"output_dir": "W:/path/to/training/model",
"output_name": "LoRA_Name",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "S:/path/to/models/flux1-dev-fp8.safetensors",
"prior_loss_weight": 1,
"random_crop": false,
"rank_dropout": 0,
"rank_dropout_scale": false,
"reg_data_dir": "",
"rescaled": false,
"resume": "",
"resume_from_huggingface": "",
"sample_every_n_epochs": 1,
"sample_every_n_steps": 0,
"sample_prompts": "an image of a sample prompt.",
"sample_sampler": "euler_a",
"save_as_bool": false,
"save_every_n_epochs": 1,
"save_every_n_steps": 0,
"save_last_n_epochs": 0,
"save_last_n_epochs_state": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 2,
"save_model_as": "safetensors",
"save_precision": "float",
"save_state": true,
"save_state_on_train_end": true,
"save_state_to_huggingface": false,
"scale_v_pred_loss_like_noise_pred": false,
"scale_weight_norms": 0,
"sdxl": false,
"sdxl_cache_text_encoder_outputs": false,
"sdxl_no_half_vae": false,
"seed": 1,
"shuffle_caption": false,
"single_blocks_to_swap": 0,
"single_dim": "",
"single_mod_dim": "",
"skip_cache_check": false,
"split_mode": false,
"split_qkv": false,
"stop_text_encoder_training": 0,
"t5xxl": "S:\\path\\to\\kohya_ss\\models\\t5xxl_fp16.safetensors",
"t5xxl_lr": 5e-05,
"t5xxl_max_token_length": 512,
"text_encoder_lr": 5e-05,
"timestep_sampling": "sigmoid",
"train_batch_size": 4,
"train_blocks": "all",
"train_data_dir": "W:/path/to/training/img",
"train_double_block_indices": "all",
"train_norm": false,
"train_on_input": true,
"train_single_block_indices": "all",
"train_t5xxl": false,
"training_comment": "Claim your LoRA",
"txt_attn_dim": "",
"txt_mlp_dim": "",
"txt_mod_dim": "",
"unet_lr": 0.0006,
"unit": 1,
"up_lr_weight": "",
"use_cp": false,
"use_scalar": false,
"use_tucker": false,
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "",
"vae_batch_size": 1,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}
What ever I do I get the same error.. My vram or ram doesn't even get a bump when I click training.. It begins to load and then errors out.
I'm running a 3060 with 12gb of vram.. 16gb of ram. If I understand things right these settings should be able to do with a lot less. I Even tryed to make the images 256*256 just to see if that would effect things and it didn't. I get the same error.. Here is the log of me starting the gui and then the training session:
I'm using the 8fp dev version of flux that 11.1 gb. And also the suggested vae and clip-l.
12:14:37-913053 INFO Kohya_ss GUI version: v24.2.0
12:14:38-768289 INFO Submodule initialized and updated.
12:14:38-772320 INFO nVidia toolkit detected
12:14:41-421978 INFO Torch 2.5.0+cu124
12:14:41-474129 INFO Torch backend: nVidia CUDA 12.4 cuDNN 90100
12:14:41-482128 INFO Torch detected GPU: NVIDIA GeForce RTX 3060 VRAM 12288MB Arch 8.6 Cores 28
12:14:41-484129 INFO Python version is 3.10.10 (tags/v3.10.10:aad5f6a, Feb 7 2023, 17:20:36) [MSC v.1929 64 bit
(AMD64)]
12:14:41-487128 INFO Installing/Validating requirements from requirements_pytorch_windows.txt...
12:14:42-408853 INFO Looking in indexes: https://pypi.org/simple, https://download.pytorch.org/whl/cu124
12:14:42-410851 INFO Obtaining file:///C:/ai/Kohya/kohya_ss/sd-scripts (from -r
C:\ai\Kohya\kohya_ss\requirements.txt (line 38))
12:14:42-413852 INFO Preparing metadata (setup.py): started
12:14:43-182505 INFO Preparing metadata (setup.py): finished with status 'done'
12:14:44-414783 INFO Installing collected packages: library
12:14:44-415781 INFO Attempting uninstall: library
12:14:44-417344 INFO Found existing installation: library 0.0.0
12:14:44-429544 INFO Uninstalling library-0.0.0:
12:14:49-279052 INFO Successfully uninstalled library-0.0.0
12:14:49-281083 INFO Running setup.py develop for library
12:14:50-625654 INFO Successfully installed library
12:14:51-227435 INFO headless: False
12:14:51-243466 INFO Using shell=True when running external commands...
* Running on local URL: http://127.0.0.1:7860
To create a public link, set `share=True` in `launch()`.
12:17:07-229122 INFO Loading config...
12:20:09-104034 INFO Start training LoRA Flux1 ...
12:20:09-106003 INFO Validating lr scheduler arguments...
12:20:09-108002 INFO Validating optimizer arguments...
12:20:09-110034 INFO Validating lora type is Flux1 if flux1 checkbox is checked...
12:20:09-111034 INFO Validating C:/ai/Kohya/kohya_ss/dataset/logs existence and writability... SUCCESS
12:20:09-113003 INFO Validating C:/ai/Kohya/kohya_ss/dataset/outputs existence and writability... SUCCESS
12:20:09-115001 INFO Validating C:/ai/test/kohya_ss/models/flux1-dev-fp8(2).safetensors existence... SUCCESS
12:20:09-117001 INFO Validating C:/ai/Kohya/kohya_ss/dataset/images/img existence... SUCCESS
12:20:09-118004 INFO Folder 40_ohwx man: 40 repeats found
12:20:09-121002 INFO Folder 40_ohwx man: 16 images found
12:20:09-123001 INFO Folder 40_ohwx man: 16 * 40 = 640 steps
12:20:09-125031 INFO Regularization factor: 1
12:20:09-126002 INFO Train batch size: 4
12:20:09-131003 INFO Gradient accumulation steps: 1
12:20:09-136003 INFO Epoch: 200
12:20:09-138002 INFO Max train steps: 1200
12:20:09-140001 INFO stop_text_encoder_training = 0
12:20:09-143003 INFO lr_warmup_steps = 0
12:20:09-146003 INFO Saving training config to
C:/ai/Kohya/kohya_ss/dataset/outputs\LoRA_Name_20250127-122009.json...
12:20:09-150003 INFO Executing command: C:\ai\Kohya\kohya_ss\venv\Scripts\accelerate.EXE launch --dynamo_backend no
--dynamo_mode default --gpu_ids 1 --mixed_precision bf16 --num_processes 1 --num_machines 1
--num_cpu_threads_per_process 1 C:/ai/Kohya/kohya_ss/sd-scripts/flux_train_network.py
--config_file C:/ai/Kohya/kohya_ss/dataset/outputs/config_lora-20250127-122009.toml
C:\ai\Kohya\kohya_ss\venv\lib\site-packages\diffusers\utils\outputs.py:63: FutureWarning: `torch.utils._pytree._register_pytree_node` is deprecated. Please use `torch.utils._pytree.register_pytree_node` instead.
torch.utils._pytree._register_pytree_node(
C:\ai\Kohya\kohya_ss\venv\lib\site-packages\diffusers\utils\outputs.py:63: FutureWarning: `torch.utils._pytree._register_pytree_node` is deprecated. Please use `torch.utils._pytree.register_pytree_node` instead.
torch.utils._pytree._register_pytree_node(
C:\ai\Kohya\kohya_ss\venv\lib\site-packages\diffusers\utils\outputs.py:63: FutureWarning: `torch.utils._pytree._register_pytree_node` is deprecated. Please use `torch.utils._pytree.register_pytree_node` instead.
torch.utils._pytree._register_pytree_node(
2025-01-27 12:20:27 INFO Loading settings from train_util.py:4625
C:/ai/Kohya/kohya_ss/dataset/outputs/config_lora-20250127-122009.toml...
INFO highvram is enabled / highvramが有効です train_util.py:4296
2025-01-27 12:20:27 INFO Checking the state dict: Diffusers or BFL, dev or schnell flux_utils.py:43
INFO t5xxl_max_token_length: 512 flux_train_network.py:152
C:\ai\Kohya\kohya_ss\venv\lib\site-packages\transformers\tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884
warnings.warn(
You are using the default legacy behaviour of the . This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565
2025-01-27 12:20:28 INFO Using DreamBooth method. train_network.py:457
INFO prepare images. train_util.py:2053
INFO get image size from name of cache files train_util.py:1944
100%|██████████████████████████████████████████████████████████████████████████████████████████| 16/16 [00:00 flux_utils.py:137
INFO Loaded fp8 FLUX model flux_train_network.py:101
INFO enable block swap: blocks_to_swap=23 flux_train_network.py:115
FLUX: Block swap enabled. Swapping 23 blocks, double blocks: 11, single blocks: 24.
INFO Building CLIP-L flux_utils.py:179
2025-01-27 12:20:29 INFO Loading state dict from C:/ai/test/kohya_ss/models/clip_l.safetensors flux_utils.py:275
2025-01-27 12:20:30 INFO Loaded CLIP-L: flux_utils.py:278
INFO Loading state dict from flux_utils.py:330
C:/ai/test/kohya_ss/models/t5xxl_fp8_e4m3fn.safetensors
INFO Loaded T5xxl: flux_utils.py:333
INFO Loaded fp8 T5XXL model flux_train_network.py:135
INFO Building AutoEncoder flux_utils.py:144
INFO Loading state dict from C:/ai/test/kohya_ss/models/ae.safetensors flux_utils.py:149
2025-01-27 12:20:31 INFO Loaded AE: flux_utils.py:152
import network module: networks.lora_flux
INFO [Dataset 0] train_util.py:2589
INFO caching latents with caching strategy. train_util.py:1097
INFO caching latents... train_util.py:1146
0%| | 0/16 [00:00
sys.exit(main())
File "C:\ai\Kohya\kohya_ss\venv\lib\site-packages\accelerate\commands\accelerate_cli.py", line 48, in main
args.func(args)
File "C:\ai\Kohya\kohya_ss\venv\lib\site-packages\accelerate\commands\launch.py", line 1106, in launch_command
simple_launcher(args)
File "C:\ai\Kohya\kohya_ss\venv\lib\site-packages\accelerate\commands\launch.py", line 704, in simple_launcher
raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd)
subprocess.CalledProcessError: Command '['C:\\ai\\Kohya\\kohya_ss\\venv\\Scripts\\python.exe', 'C:/ai/Kohya/kohya_ss/sd-scripts/flux_train_network.py', '--config_file', 'C:/ai/Kohya/kohya_ss/dataset/outputs/config_lora-20250127-122009.toml']' returned non-zero exit status 3221225477.
12:21:23-401983 INFO Training has ended.
Contributor guide
No contributing guide indexed for this repository
Research direction
Reproduce the Windows launch command for sd-scripts/flux_train_network.py using the generated config_lora-20250127-122009.toml and the reported Flux checkpoint. Start by tracing the startup path through train_util.py and flux_utils.py, then capture the complete subprocess output beyond the truncated log. Done means identifying the cause of exit status 3221225477 and documenting or fixing the failure so training begins.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- python
- Domain
- machine-learning
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 25/100