ask for help

First of all, thank you for your open-source code

I used sdxl as the base model and fine tuned my model on my own data. Later, I downloaded a batch of image-text pair data and want to continue training on the previously saved model. My training command is as follows（It has already been run in advance finetune/merge_captions_to_metadata.py and finetune/prepare_buckets_latents.py preprocessing scripts）:

nohup accelerate launch --num_cpu_threads_per_process=8 "./sdxl_train.py" --full_bf16 --pretrained_model_name_or_path="/home/yb/stable-diffusion-webui/models/Stable-diffusion/sd_xl_base_1.0.safetensors" --in_json="/mnt/data2/mjdata/1025_0105/meta_lat.json" --train_data_dir="/mnt/data2/mjdata/1025_0105" --output_dir="/home/yb/stable-diffusion-webui/models/Stable-diffusion/smalljourney_105w_add_66w" --dataset_repeats=1 --learning_rate=2e-06 --enable_bucket --resolution="1024,1024" --min_bucket_reso=256 --max_bucket_reso=1024 --save_model_as=safetensors --output_name="smalljourney_105w_add_66w_xl" --max_token_length=225 --cache_text_encoder_outputs --cache_text_encoder_outputs_to_disk --no_half_vae --learning_rate=2e-06 --lr_scheduler="constant" --train_batch_size="48" --max_train_steps="84" --save_every_n_epochs="1" --mixed_precision="bf16" --save_precision="fp16" --seed="20232022" --cache_latents --cache_latents_to_disk --optimizer_type="AdamW8bit" --max_train_epochs=20 --max_data_loader_n_workers="24" --max_token_length=225 --vae_batch_size="48" --bucket_reso_steps=32 --save_every_n_steps="5000" --save_state --gradient_checkpointing --xformers --noise_offset=0.1 --adaptive_noise_scale=0.01 --sample_sampler=euler_a --sample_prompts="/home/yb/stable-diffusion-webui/models/Stable-diffusion/sample/prompt.txt" --sample_every_n_steps="100" --resume="/home/yb/stable-diffusion-webui/models/Stable-diffusion/smalljourney_105w_xl-000009-state" &

Then I received the following error message：
IndexError: shape mismatch: indexing tensors could not be broadcast together with shapes [144], [145]

All log outputs are as follows：
prepare accelerator
loading model for process 0/1
load StableDiffusion checkpoint: /home/yb/stable-diffusion-webui/models/Stable-diffusion/sd_xl_base_1.0.safetensors
building U-Net
loading U-Net from checkpoint
U-Net:
building text encoders
loading text encoders from checkpoint
text encoder 1:
text encoder 2:
building VAE
loading VAE from checkpoint
VAE:
Disable Diffusers' xformers
Enable xformers for U-Net
[Dataset 0]
caching latents.
checking cache validity...
100%|██████████| 662552/662552 [00:00<00:00, 1667230.32it/s]
caching latents...
0it [00:00, ?it/s]
[Dataset 0]
caching text encoder outputs.
checking cache existence...
100%|██████████| 662552/662552 [00:02<00:00, 327565.23it/s]
caching text encoder outputs...
24%|██▎ | 3278/13804 [1:02:40<3:21:15, 1.15s/it]
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /home/yb/kohya_ss/./sdxl_train.py:652 in │
│ │
│ 649 │ print("args:") │
│ 650 │ print(args) │
│ 651 │ │
│ ❱ 652 │ train(args) │
│ 653 │
│ │
│ /home/yb/kohya_ss/./sdxl_train.py:219 in train │
│ │
│ 216 │ │ if args.cache_text_encoder_outputs: │
│ 217 │ │ │ # Text Encodes are eval and no grad │
│ 218 │ │ │ with torch.no_grad(): │
│ ❱ 219 │ │ │ │ train_dataset_group.cache_text_encoder_outputs( │
│ 220 │ │ │ │ │ (tokenizer1, tokenizer2), │
│ 221 │ │ │ │ │ (text_encoder1, text_encoder2), │
│ 222 │ │ │ │ │ accelerator.device, │
│ │
│ /home/yb/kohya_ss/library/train_util.py:1828 in cache_text_encoder_outputs │
│ │
│ 1825 │ ): │
│ 1826 │ │ for i, dataset in enumerate(self.datasets): │
│ 1827 │ │ │ print(f"[Dataset {i}]") │
│ ❱ 1828 │ │ │ dataset.cache_text_encoder_outputs(tokenizers, text_encod │
│ 1829 │ │
│ 1830 │ def set_caching_mode(self, caching_mode): │
│ 1831 │ │ for dataset in self.datasets: │
│ │
│ /home/yb/kohya_ss/library/train_util.py:931 in cache_text_encoder_outputs │
│ │
│ 928 │ │ │ infos, input_ids1, input_ids2 = zip(*batch) │
│ 929 │ │ │ input_ids1 = torch.stack(input_ids1, dim=0) │
│ 930 │ │ │ input_ids2 = torch.stack(input_ids2, dim=0) │
│ ❱ 931 │ │ │ cache_batch_text_encoder_outputs( │
│ 932 │ │ │ │ infos, tokenizers, text_encoders, self.max_token_leng │
│ 933 │ │ │ ) │
│ 934 │
│ │
│ /home/yb/kohya_ss/library/train_util.py:2178 in │
│ cache_batch_text_encoder_outputs │
│ │
│ 2175 │ input_ids2 = input_ids2.to(text_encoders[1].device) │
│ 2176 │ │
│ 2177 │ with torch.no_grad(): │
│ ❱ 2178 │ │ b_hidden_state1, b_hidden_state2, b_pool2 = get_hidden_states │
│ 2179 │ │ │ max_token_length, │
│ 2180 │ │ │ input_ids1, │
│ 2181 │ │ │ input_ids2, │
│ │
│ /home/yb/kohya_ss/library/train_util.py:3926 in get_hidden_states_sdxl │
│ │
│ 3923 │ hidden_states2 = enc_out["hidden_states"][-2] # penuultimate lay │
│ 3924 │ │
│ 3925 │ # pool2 = enc_out["text_embeds"] │
│ ❱ 3926 │ pool2 = pool_workaround(text_encoder2, enc_out["last_hidden_state │
│ 3927 │ │
│ 3928 │ # b*n, 77, 768 or 1280 -> b, n*77, 768 or 1280 │
│ 3929 │ n_size = 1 if max_token_length is None else max_token_length // 7 │
│ │
│ /home/yb/kohya_ss/library/train_util.py:3893 in pool_workaround │
│ │
│ 3890 │ eos_token_index = eos_token_index.to(device=last_hidden_state.dev │
│ 3891 │ │
│ 3892 │ # get hidden states for EOS token │
│ ❱ 3893 │ pooled_output = last_hidden_state[torch.arange(last_hidden_state. │
│ 3894 │ │
│ 3895 │ # apply projection: projection may be of different dtype than las │
│ 3896 │ pooled_output = text_encoder.text_projection(pooled_output.to(tex │
╰──────────────────────────────────────────────────────────────────────────────╯
IndexError: shape mismatch: indexing tensors could not be broadcast together
with shapes [144], [145]
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /home/yb/kohya_ss/venv/bin/accelerate:8 in │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /home/yb/kohya_ss/venv/lib/python3.10/site-packages/accelerate/commands/acce │
│ lerate_cli.py:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if __name__ == "__main__": │
│ │
│ /home/yb/kohya_ss/venv/lib/python3.10/site-packages/accelerate/commands/laun │
│ ch.py:918 in launch_command │
│ │
│ 915 │ elif defaults is not None and defaults.compute_environment == Comp │
│ 916 │ │ sagemaker_launcher(defaults, args) │
│ 917 │ else: │
│ ❱ 918 │ │ simple_launcher(args) │
│ 919 │
│ 920 │
│ 921 def main(): │
│ │
│ /home/yb/kohya_ss/venv/lib/python3.10/site-packages/accelerate/commands/laun │
│ ch.py:580 in simple_launcher │
│ │
│ 577 │ process.wait() │
│ 578 │ if process.returncode != 0: │
│ 579 │ │ if not args.quiet: │
│ ❱ 580 │ │ │ raise subprocess.CalledProcessError(returncode=process.ret │
│ 581 │ │ else: │
│ 582 │ │ │ sys.exit(1) │
│ 583 │
╰──────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['/home/yb/kohya_ss/venv/bin/python',
'./sdxl_train.py', '--full_bf16',
'--pretrained_model_name_or_path=/home/yb/stable-diffusion-webui/models/Stable-d
iffusion/sd_xl_base_1.0.safetensors',
'--in_json=/mnt/data2/mjdata/1025_0105/meta_lat.json',
'--train_data_dir=/mnt/data2/mjdata/1025_0105',
'--output_dir=/home/yb/stable-diffusion-webui/models/Stable-diffusion/smalljourn
ey_105w_add_66w', '--dataset_repeats=1', '--learning_rate=2e-06',
'--enable_bucket', '--resolution=1024,1024', '--min_bucket_reso=256',
'--max_bucket_reso=1024', '--save_model_as=safetensors',
'--output_name=smalljourney_105w_add_66w_xl', '--max_token_length=225',
'--cache_text_encoder_outputs', '--cache_text_encoder_outputs_to_disk',
'--no_half_vae', '--learning_rate=2e-06', '--lr_scheduler=constant',
'--train_batch_size=48', '--max_train_steps=84', '--save_every_n_epochs=1',
'--mixed_precision=bf16', '--save_precision=fp16', '--seed=20232022',
'--cache_latents', '--cache_latents_to_disk', '--optimizer_type=AdamW8bit',
'--max_train_epochs=20', '--max_data_loader_n_workers=24',
'--max_token_length=225', '--vae_batch_size=48', '--bucket_reso_steps=32',
'--save_every_n_steps=5000', '--save_state', '--gradient_checkpointing',
'--xformers', '--noise_offset=0.1', '--adaptive_noise_scale=0.01',
'--sample_sampler=euler_a',
'--sample_prompts=/home/yb/stable-diffusion-webui/models/Stable-diffusion/sample
/prompt.txt', '--sample_every_n_steps=100',
'--resume=/home/yb/stable-diffusion-webui/models/Stable-diffusion/smalljourney_1
05w_xl-000009-state']' returned non-zero exit status 1.

I don't know how to locate and resolve the above error, I hope to receive your reply
thanks！

Start with sdxl_train.py and the traceback path in library/train_util.py, especially cache_text_encoder_outputs, get_hidden_states_sdxl, and pool_workaround. Reproduce the reported SDXL text-encoder caching failure and determine why the indexing tensors have lengths 144 and 145; done means the cause and a verified correction are identified.

Written by the indexing model from the issue text.