RuntimeError: CUDA error: invalid argument

I'm getting **"RuntimeError: CUDA error: invalid argument"** error, and the script reports **training success** even when failing.
I posted the logs below. Can you please help make it work?

```javascript
2023-02-28 09:23:39,877 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)
2023-02-28 09:23:39,906 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)
2023-02-28 09:23:39,933 sagemaker-training-toolkit INFO No Neurons detected (normal if no neurons installed)
2023-02-28 09:23:39,944 sagemaker-training-toolkit INFO Invoking user script
Training Env:
{
"additional_framework_parameters": {},
"channel_input_dirs": {
"images": "/opt/ml/input/data/images"
},
"current_host": "algo-1",
"current_instance_group": "homogeneousCluster",
"current_instance_group_hosts": [
"algo-1"
],
"current_instance_type": "ml.g5.8xlarge",
"distribution_hosts": [],
"distribution_instance_groups": [],
"framework_module": null,
"hosts": [
"algo-1"
],
"hyperparameters": {
"attention": "xformers",
"class_data_dir": "/opt/ml/input/data/class_images/",
"class_prompt": "photo of a dog",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"instance_data_dir": "/opt/ml/input/data/images/",
"instance_prompt": "photo of zwx dog",
"learning_rate": 2e-06,
"lr_scheduler": "constant",
"lr_warmup_steps": 0,
"max_train_steps": 300,
"mixed_precision": "fp16",
"model_name": "aws-trained-dreambooth-model",
"models_path": "/opt/ml/model/",
"not_cache_latents": true,
"num_class_images": 50,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 0.5,
"resolution": 512,
"sample_batch_size": 1,
"save_steps": 300,
"save_use_epochs": false,
"train_batch_size": 1,
"train_text_encoder": false,
"use_8bit_adam": false,
"use_ema": true,
"with_prior_preservation": true
},
"input_config_dir": "/opt/ml/input/config",
"input_data_config": {
"images": {
"TrainingInputMode": "File",
"S3DistributionType": "FullyReplicated",
"RecordWrapperType": "None"
}
},
"input_dir": "/opt/ml/input",
"instance_groups": [
"homogeneousCluster"
],
"instance_groups_dict": {
"homogeneousCluster": {
"instance_group_name": "homogeneousCluster",
"instance_type": "ml.g5.8xlarge",
"hosts": [
"algo-1"
]
}
},
"is_hetero": false,
"is_master": true,
"is_modelparallel_enabled": null,
"is_smddpmprun_installed": false,
"job_name": "sd-dreambooth-finetuning-v2-2023-02-28-09-15-28-012",
"log_level": 20,
"master_hostname": "algo-1",
"model_dir": "/opt/ml/model",
"module_dir": "/opt/ml/code",
"module_name": "train_dreambooth",
"network_interface_name": "eth0",
"num_cpus": 32,
"num_gpus": 1,
"num_neurons": 0,
"output_data_dir": "/opt/ml/output/data",
"output_dir": "/opt/ml/output",
"output_intermediate_dir": "/opt/ml/output/intermediate",
"resource_config": {
"current_host": "algo-1",
"current_instance_type": "ml.g5.8xlarge",
"current_group_name": "homogeneousCluster",
"hosts": [
"algo-1"
],
"instance_groups": [
{
"instance_group_name": "homogeneousCluster",
"instance_type": "ml.g5.8xlarge",
"hosts": [
"algo-1"
]
}
],
"network_interface_name": "eth0"
},
"user_entry_point": "train_dreambooth.py"
}
Environment variables:
SM_HOSTS=["algo-1"]
SM_NETWORK_INTERFACE_NAME=eth0
SM_HPS=
{
"attention": "xformers",
"class_data_dir": "/opt/ml/input/data/class_images/",
"class_prompt": "photo of a dog",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"instance_data_dir": "/opt/ml/input/data/images/",
"instance_prompt": "photo of zwx dog",
"learning_rate": 0.000002,
"lr_scheduler": "constant",
"lr_warmup_steps": 0,
"max_train_steps": 300,
"mixed_precision": "fp16",
"model_name": "aws-trained-dreambooth-model",
"models_path": "/opt/ml/model/",
"not_cache_latents": true,
"num_class_images": 50,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 0.5,
"resolution": 512,
"sample_batch_size": 1,
"save_steps": 300,
"save_use_epochs": false,
"train_batch_size": 1,
"train_text_encoder": false,
"use_8bit_adam": false,
"use_ema": true,
"with_prior_preservation": true
}
SM_USER_ENTRY_POINT=train_dreambooth.py
SM_FRAMEWORK_PARAMS=
{}
SM_RESOURCE_CONFIG=
{
"current_group_name": "homogeneousCluster",
"current_host": "algo-1",
"current_instance_type": "ml.g5.8xlarge",
"hosts": [
"algo-1"
],
"instance_groups": [
{
"hosts": [
"algo-1"
],
"instance_group_name": "homogeneousCluster",
"instance_type": "ml.g5.8xlarge"
}
],
"network_interface_name": "eth0"
}
SM_INPUT_DATA_CONFIG=
{
"images": {
"RecordWrapperType": "None",
"S3DistributionType": "FullyReplicated",
"TrainingInputMode": "File"
}
}
SM_OUTPUT_DATA_DIR=/opt/ml/output/data
SM_CHANNELS=["images"]
SM_CURRENT_HOST=algo-1
SM_CURRENT_INSTANCE_TYPE=ml.g5.8xlarge
SM_CURRENT_INSTANCE_GROUP=homogeneousCluster
SM_CURRENT_INSTANCE_GROUP_HOSTS=["algo-1"]
SM_INSTANCE_GROUPS=["homogeneousCluster"]
SM_INSTANCE_GROUPS_DICT=
{
"homogeneousCluster": {
"hosts": [
"algo-1"
],
"instance_group_name": "homogeneousCluster",
"instance_type": "ml.g5.8xlarge"
}
}
SM_DISTRIBUTION_INSTANCE_GROUPS=[]
SM_IS_HETERO=false
SM_MODULE_NAME=train_dreambooth
SM_LOG_LEVEL=20
SM_FRAMEWORK_MODULE=
SM_INPUT_DIR=/opt/ml/input
SM_INPUT_CONFIG_DIR=/opt/ml/input/config
SM_OUTPUT_DIR=/opt/ml/output
SM_NUM_CPUS=32
SM_NUM_GPUS=1
SM_NUM_NEURONS=0
SM_MODEL_DIR=/opt/ml/model
SM_MODULE_DIR=/opt/ml/code
SM_TRAINING_ENV=
{
"additional_framework_parameters": {},
"channel_input_dirs": {
"images": "/opt/ml/input/data/images"
},
"current_host": "algo-1",
"current_instance_group": "homogeneousCluster",
"current_instance_group_hosts": [
"algo-1"
],
"current_instance_type": "ml.g5.8xlarge",
"distribution_hosts": [],
"distribution_instance_groups": [],
"framework_module": null,
"hosts": [
"algo-1"
],
"hyperparameters": {
"attention": "xformers",
"class_data_dir": "/opt/ml/input/data/class_images/",
"class_prompt": "photo of a dog",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": true,
"instance_data_dir": "/opt/ml/input/data/images/",
"instance_prompt": "photo of zwx dog",
"learning_rate": 0.000002,
"lr_scheduler": "constant",
"lr_warmup_steps": 0,
"max_train_steps": 300,
"mixed_precision": "fp16",
"model_name": "aws-trained-dreambooth-model",
"models_path": "/opt/ml/model/",
"not_cache_latents": true,
"num_class_images": 50,
"pretrained_model_name_or_path": "runwayml/stable-diffusion-v1-5",
"prior_loss_weight": 0.5,
"resolution": 512,
"sample_batch_size": 1,
"save_steps": 300,
"save_use_epochs": false,
"train_batch_size": 1,
"train_text_encoder": false,
"use_8bit_adam": false,
"use_ema": true,
"with_prior_preservation": true
},
"input_config_dir": "/opt/ml/input/config",
"input_data_config": {
"images": {
"RecordWrapperType": "None",
"S3DistributionType": "FullyReplicated",
"TrainingInputMode": "File"
}
},
"input_dir": "/opt/ml/input",
"instance_groups": [
"homogeneousCluster"
],
"instance_groups_dict": {
"homogeneousCluster": {
"hosts": [
"algo-1"
],
"instance_group_name": "homogeneousCluster",
"instance_type": "ml.g5.8xlarge"
}
},
"is_hetero": false,
"is_master": true,
"is_modelparallel_enabled": null,
"is_smddpmprun_installed": false,
"job_name": "sd-dreambooth-finetuning-v2-2023-02-28-09-15-28-012",
"log_level": 20,
"master_hostname": "algo-1",
"model_dir": "/opt/ml/model",
"module_dir": "/opt/ml/code",
"module_name": "train_dreambooth",
"network_interface_name": "eth0",
"num_cpus": 32,
"num_gpus": 1,
"num_neurons": 0,
"output_data_dir": "/opt/ml/output/data",
"output_dir": "/opt/ml/output",
"output_intermediate_dir": "/opt/ml/output/intermediate",
"resource_config": {
"current_group_name": "homogeneousCluster",
"current_host": "algo-1",
"current_instance_type": "ml.g5.8xlarge",
"hosts": [
"algo-1"
],
"instance_groups": [
{
"hosts": [
"algo-1"
],
"instance_group_name": "homogeneousCluster",
"instance_type": "ml.g5.8xlarge"
}
],
"network_interface_name": "eth0"
},
"user_entry_point": "train_dreambooth.py"
}
SM_USER_ARGS=["--attention","xformers","--class_data_dir","/opt/ml/input/data/class_images/","--class_prompt","photo of a dog","--gradient_accumulation_steps","1","--gradient_checkpointing","True","--instance_data_dir","/opt/ml/input/data/images/","--instance_prompt","photo of zwx dog","--learning_rate","2e-06","--lr_scheduler","constant","--lr_warmup_steps","0","--max_train_steps","300","--mixed_precision","fp16","--model_name","aws-trained-dreambooth-model","--models_path","/opt/ml/model/","--not_cache_latents","True","--num_class_images","50","--pretrained_model_name_or_path","runwayml/stable-diffusion-v1-5","--prior_loss_weight","0.5","--resolution","512","--sample_batch_size","1","--save_steps","300","--save_use_epochs","False","--train_batch_size","1","--train_text_encoder","False","--use_8bit_adam","False","--use_ema","True","--with_prior_preservation","True"]
SM_OUTPUT_INTERMEDIATE_DIR=/opt/ml/output/intermediate
SM_CHANNEL_IMAGES=/opt/ml/input/data/images
SM_HP_ATTENTION=xformers
SM_HP_CLASS_DATA_DIR=/opt/ml/input/data/class_images/
SM_HP_CLASS_PROMPT=photo of a dog
SM_HP_GRADIENT_ACCUMULATION_STEPS=1
SM_HP_GRADIENT_CHECKPOINTING=true
SM_HP_INSTANCE_DATA_DIR=/opt/ml/input/data/images/
SM_HP_INSTANCE_PROMPT=photo of zwx dog
SM_HP_LEARNING_RATE=2e-06
SM_HP_LR_SCHEDULER=constant
SM_HP_LR_WARMUP_STEPS=0
SM_HP_MAX_TRAIN_STEPS=300
SM_HP_MIXED_PRECISION=fp16
SM_HP_MODEL_NAME=aws-trained-dreambooth-model
SM_HP_MODELS_PATH=/opt/ml/model/
SM_HP_NOT_CACHE_LATENTS=true
SM_HP_NUM_CLASS_IMAGES=50
SM_HP_PRETRAINED_MODEL_NAME_OR_PATH=runwayml/stable-diffusion-v1-5
SM_HP_PRIOR_LOSS_WEIGHT=0.5
SM_HP_RESOLUTION=512
SM_HP_SAMPLE_BATCH_SIZE=1
SM_HP_SAVE_STEPS=300
SM_HP_SAVE_USE_EPOCHS=false
SM_HP_TRAIN_BATCH_SIZE=1
SM_HP_TRAIN_TEXT_ENCODER=false
SM_HP_USE_8BIT_ADAM=false
SM_HP_USE_EMA=true
SM_HP_WITH_PRIOR_PRESERVATION=true
PYTHONPATH=/opt/ml/code:/opt/conda/bin:/opt/conda/lib/python39.zip:/opt/conda/lib/python3.9:/opt/conda/lib/python3.9/lib-dynload:/opt/conda/lib/python3.9/site-packages:/opt/ml/code/repositories/xformers
Invoking script with the following command:
/opt/conda/bin/python train_dreambooth.py --attention xformers --class_data_dir /opt/ml/input/data/class_images/ --class_prompt photo of a dog --gradient_accumulation_steps 1 --gradient_checkpointing True --instance_data_dir /opt/ml/input/data/images/ --instance_prompt photo of zwx dog --learning_rate 2e-06 --lr_scheduler constant --lr_warmup_steps 0 --max_train_steps 300 --mixed_precision fp16 --model_name aws-trained-dreambooth-model --models_path /opt/ml/model/ --not_cache_latents True --num_class_images 50 --pretrained_model_name_or_path runwayml/stable-diffusion-v1-5 --prior_loss_weight 0.5 --resolution 512 --sample_batch_size 1 --save_steps 300 --save_use_epochs False --train_batch_size 1 --train_text_encoder False --use_8bit_adam False --use_ema True --with_prior_preservation True
2023-02-28 09:23:39,944 sagemaker-training-toolkit INFO Exceptions not imported for SageMaker Debugger as it is not installed.
2023-02-28 09:23:39,945 sagemaker-training-toolkit INFO Exceptions not imported for SageMaker TF as Tensorflow is not installed.
/opt/conda/lib/python3.9/site-packages/transformers/generation_utils.py:24: FutureWarning: Importing `GenerationMixin` from `src/transformers/generation_utils.py` is deprecated and will be removed in Transformers v5. Import as `from transformers import GenerationMixin` instead.
warnings.warn(
Patching transformers to fix kwargs errors.
{'model_name': 'aws-trained-dreambooth-model', 'models_path': '/opt/ml/model/', 'use_lora': False, 'use_cpu': False, 'lora_models_path': None, 'pretrained_model_name_or_path': 'runwayml/stable-diffusion-v1-5', 'pretrained_vae_name_or_path': None, 'revision': None, 'tokenizer_name': None, 'instance_data_dir': '/opt/ml/input/data/images/', 'class_data_dir': '/opt/ml/input/data/class_images/', 'instance_prompt': 'photo of zwx dog', 'class_prompt': 'photo of a dog', 'pad_tokens': False, 'with_prior_preservation': True, 'save_use_global_counts': False, 'save_use_epochs': True, 'prior_loss_weight': 0.5, 'num_class_images': 50, 'output_dir': 'text-inversion-model', 'seed': -1, 'resolution': 512, 'center_crop': False, 'train_text_encoder': 'False', 'train_batch_size': 1, 'sample_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 300, 'epoch': 0, 'save_steps': 300, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 2e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'use_8bit_adam': True, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'push_to_hub': False, 'hub_token': None, 'hub_model_id': None, 'logging_dir': 'logs', 'mixed_precision': 'fp16', 'not_cache_latents': 'True', 'hflip': False, 'local_rank': -1, 'concepts_list': [{'instance_prompt': 'photo of zwx dog', 'class_prompt': 'photo of a dog', 'instance_data_dir': '/opt/ml/input/data/images/', 'class_data_dir': '/opt/ml/input/data/class_images/', 'num_class_images': 50, 'instance_token': '', 'class_token': '', 'class_negative_prompt': '', 'class_guidance_scale': 7.5, 'class_infer_steps': 60}], 'use_ema': True, 'max_token_length': 75, 'half_model': False, 'attention': 'xformers', 'shuffle_tags': False}
Replace CrossAttention.forward to use xformers
Checking concept: {'instance_prompt': 'photo of zwx dog', 'class_prompt': 'photo of a dog', 'instance_data_dir': '/opt/ml/input/data/images/', 'class_data_dir': '/opt/ml/input/data/class_images/', 'num_class_images': 50, 'instance_token': '', 'class_token': '', 'class_negative_prompt': '', 'class_guidance_scale': 7.5, 'class_infer_steps': 60}
Concept requires 50 images.
Class dir /opt/ml/input/data/class_images has 0 images.
#015Downloading (…)_model.safetensors";: 0%| | 0.00/335M [00:00