untrainable ERROR need help thanks 1

╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /mnt/workspace/lora-scripts/./sd-scripts/train_network.py:873 in │
│ │
│ 870 │ args = parser.parse_args() │
│ 871 │ args = train_util.read_config_from_file(args, parser) │
│ 872 │ │
│ ❱ 873 │ train(args) │
│ 874 │
│ │
│ /mnt/workspace/lora-scripts/./sd-scripts/train_network.py:688 in train │
│ │
│ 685 │ │ │ │ │ params_to_clip = network.get_trainable_params() │
│ 686 │ │ │ │ │ accelerator.clip_grad_norm_(params_to_clip, args.m │
│ 687 │ │ │ │ │
│ ❱ 688 │ │ │ │ optimizer.step() │
│ 689 │ │ │ │ lr_scheduler.step() │
│ 690 │ │ │ │ optimizer.zero_grad(set_to_none=True) │
│ 691 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/optimizer.py:134 in step │
│ │
│ 131 │ │ │ │ xm.optimizer_step(self.optimizer, optimizer_args=optim │
│ 132 │ │ │ elif self.scaler is not None: │
│ 133 │ │ │ │ scale_before = self.scaler.get_scale() │
│ ❱ 134 │ │ │ │ self.scaler.step(self.optimizer, closure) │
│ 135 │ │ │ │ self.scaler.update() │
│ 136 │ │ │ │ scale_after = self.scaler.get_scale() │
│ 137 │ │ │ │ # If we reduced the loss scale, it means the optimizer │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/cuda/amp/grad_scaler.py:338 in │
│ step │
│ │
│ 335 │ │ │
│ 336 │ │ assert len(optimizer_state["found_inf_per_device"]) > 0, "No i │
│ 337 │ │ │
│ ❱ 338 │ │ retval = self._maybe_opt_step(optimizer, optimizer_state, *arg │
│ 339 │ │ │
│ 340 │ │ optimizer_state["stage"] = OptState.STEPPED │
│ 341 │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/cuda/amp/grad_scaler.py:285 in │
│ _maybe_opt_step │
│ │
│ 282 │ def _maybe_opt_step(self, optimizer, optimizer_state, *args, **kwa │
│ 283 │ │ retval = None │
│ 284 │ │ if not sum(v.item() for v in optimizer_state["found_inf_per_de │
│ ❱ 285 │ │ │ retval = optimizer.step(*args, **kwargs) │
│ 286 │ │ return retval │
│ 287 │ │
│ 288 │ def step(self, optimizer, *args, **kwargs): │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/optim/lr_scheduler.py:65 in │
│ wrapper │
│ │
│ 62 │ │ │ │ instance = instance_ref() │
│ 63 │ │ │ │ instance._step_count += 1 │
│ 64 │ │ │ │ wrapped = func.__get__(instance, cls) │
│ ❱ 65 │ │ │ │ return wrapped(*args, **kwargs) │
│ 66 │ │ │ │
│ 67 │ │ │ # Note that the returned function here is no longer a bou │
│ 68 │ │ │ # so attributes like `__func__` and `__self__` no longer │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/optim/optimizer.py:113 in │
│ wrapper │
│ │
│ 110 │ │ │ │ obj, *_ = args │
│ 111 │ │ │ │ profile_name = "Optimizer.step#{}.step".format(obj.__c │
│ 112 │ │ │ │ with torch.autograd.profiler.record_function(profile_n │
│ ❱ 113 │ │ │ │ │ return func(*args, **kwargs) │
│ 114 │ │ │ return wrapper │
│ 115 │ │ │
│ 116 │ │ hooked = getattr(self.__class__.step, "hooked", None) │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/autograd/grad_mode.py:27 in │
│ decorate_context │
│ │
│ 24 │ │ @functools.wraps(func) │
│ 25 │ │ def decorate_context(*args, **kwargs): │
│ 26 │ │ │ with self.clone(): │
│ ❱ 27 │ │ │ │ return func(*args, **kwargs) │
│ 28 │ │ return cast(F, decorate_context) │
│ 29 │ │
│ 30 │ def _wrap_generator(self, func): │
│ │
│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/optim/optimizer.py:265 │
│ in step │
│ │
│ 262 │ │ │ │ if len(state) == 0: │
│ 263 │ │ │ │ │ self.init_state(group, p, gindex, pindex) │
│ 264 │ │ │ │ │
│ ❱ 265 │ │ │ │ self.update_step(group, p, gindex, pindex) │
│ 266 │ │ │
│ 267 │ │ return loss │
│ 268 │
│ │
│ /usr/local/lib/python3.10/dist-packages/torch/autograd/grad_mode.py:27 in │
│ decorate_context │
│ │
│ 24 │ │ @functools.wraps(func) │
│ 25 │ │ def decorate_context(*args, **kwargs): │
│ 26 │ │ │ with self.clone(): │
│ ❱ 27 │ │ │ │ return func(*args, **kwargs) │
│ 28 │ │ return cast(F, decorate_context) │
│ 29 │ │
│ 30 │ def _wrap_generator(self, func): │
│ │
│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/optim/optimizer.py:506 │
│ in update_step │
│ │
│ 503 │ │ │ state["max1"], state["new_max1"] = state["new_max1"], stat │
│ 504 │ │ │ state["max2"], state["new_max2"] = state["new_max2"], stat │
│ 505 │ │ elif state["state1"].dtype == torch.uint8 and config["block_wi │
│ ❱ 506 │ │ │ F.optimizer_update_8bit_blockwise( │
│ 507 │ │ │ │ self.optimizer_name, │
│ 508 │ │ │ │ grad, │
│ 509 │ │ │ │ p, │
│ │
│ /usr/local/lib/python3.10/dist-packages/bitsandbytes/functional.py:858 in │
│ optimizer_update_8bit_blockwise │
│ │
│ 855 ) -> None: │
│ 856 │ │
│ 857 │ if g.dtype == torch.float32 and state1.dtype == torch.uint8: │
│ ❱ 858 │ │ str2optimizer8bit_blockwise[optimizer_name][0]( │
│ 859 │ │ │ get_ptr(p), │
│ 860 │ │ │ get_ptr(g), │
│ 861 │ │ │ get_ptr(state1), │
╰──────────────────────────────────────────────────────────────────────────────╯
NameError: name 'str2optimizer8bit_blockwise' is not defined
steps: 0%| | 0/3000 [00:01 │
│ │
│ 5 from accelerate.commands.accelerate_cli import main │
│ 6 if __name__ == '__main__': │
│ 7 │ sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) │
│ ❱ 8 │ sys.exit(main()) │
│ 9 │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/accelerate_cli.p │
│ y:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if __name__ == "__main__": │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:1104 │
│ in launch_command │
│ │
│ 1101 │ elif defaults is not None and defaults.compute_environment == Com │
│ 1102 │ │ sagemaker_launcher(defaults, args) │
│ 1103 │ else: │
│ ❱ 1104 │ │ simple_launcher(args) │
│ 1105 │
│ 1106 │
│ 1107 def main(): │
│ │
│ /usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py:567 in │
│ simple_launcher │
│ │
│ 564 │ process = subprocess.Popen(cmd, env=current_env) │
│ 565 │ process.wait() │
│ 566 │ if process.returncode != 0: │
│ ❱ 567 │ │ raise subprocess.CalledProcessError(returncode=process.return │
│ 568 │
│ 569 │
│ 570 def multi_gpu_launcher(args): │
╰──────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['/usr/bin/python3',
'./sd-scripts/train_network.py', '--enable_bucket',
'--pretrained_model_name_or_path=./sd-models/Deliberate_v2.safetensors',
'--train_data_dir=./train/hoodie', '--output_dir=./output',
'--logging_dir=./logs', '--resolution=512,512',
'--network_module=networks.lora', '--max_train_epochs=10',
'--learning_rate=1e-4', '--unet_lr=1e-4', '--text_encoder_lr=1e-5',
'--network_dim=32', '--network_alpha=32', '--output_name=aki',
'--lr_scheduler=cosine_with_restarts', '--train_batch_size=1',
'--save_every_n_epochs=2', '--mixed_precision=fp16', '--save_precision=fp16',
'--seed=1337', '--cache_latents', '--clip_skip=2', '--prior_loss_weight=1',
'--max_token_length=225', '--caption_extension=.txt',
'--save_model_as=safetensors', '--shuffle_caption', '--use_8bit_adam']' returned
non-zero exit status 1.
2023-07-19 13:36:52.275423: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-07-19 13:36:52.488178: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2023-07-19 13:36:53.141691: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64
2023-07-19 13:36:53.141754: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64
2023-07-19 13:36:53.141765: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
[13:36:55] WARNING The following values were not passed to ]8;id=643642;file:///usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py\launch.py]8;;\:]8;id=934749;file:///usr/local/lib/python3.10/dist-packages/accelerate/commands/launch.py#1088\1088]8;;\
`accelerate launch` and had defaults used
instead:
`--num_processes` was set to a value
of `1`
`--num_machines` was set to a value
of `1`
`--mixed_precision` was set to a
value of `'no'`
`--dynamo_backend` was set to a value
of `'no'`
To avoid this warning pass in values for each
of the problematic parameters or run
`accelerate config`.
2023-07-19 13:36:55.874734: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX2 AVX512F FMA
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2023-07-19 13:36:56.084686: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
2023-07-19 13:36:56.730278: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64
2023-07-19 13:36:56.730346: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/nvidia/lib:/usr/local/nvidia/lib64
2023-07-19 13:36:56.730357: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.
WARNING[XFORMERS]: xFormers can't load C++/CUDA extensions. xFormers was built for:
PyTorch 1.13.1+cu117 with CUDA 1107 (you have 1.12.1+cu116)
Python 3.10.9 (you have 3.10.6)
Please reinstall xformers (see https://github.com/facebookresearch/xformers#installing-xformers)
Memory-efficient attention, SwiGLU, sparse and more won't be available.
Set XFORMERS_MORE_DETAILS=1 for more details
prepare tokenizer
update token length: 225
Using DreamBooth method.
ignore directory without repeats / 繰り返し回数のないディレクトリを無視します: .ipynb_checkpoints
prepare images.
found directory train/hoodie/10_hoodie contains 30 image files
300 train images with repeating.
0 reg images.
no regularization images / 正則化画像が見つかりませんでした
[Dataset 0]
batch_size: 1
resolution: (512, 512)
enable_bucket: True
min_bucket_reso: 256
max_bucket_reso: 1024
bucket_reso_steps: 64
bucket_no_upscale: False

[Subset 0 of Dataset 0]
image_dir: "train/hoodie/10_hoodie"
image_count: 30
num_repeats: 10
shuffle_caption: True
keep_tokens: 0
caption_dropout_rate: 0.0
caption_dropout_every_n_epoches: 0
caption_tag_dropout_rate: 0.0
color_aug: False
flip_aug: False
face_crop_aug_range: None
random_crop: False
token_warmup_min: 1,
token_warmup_step: 0,
is_reg: False
class_tokens: hoodie
caption_extension: .txt

[Dataset 0]
loading image sizes.
100%|█████████████████████████████████████████| 30/30 [00:00<00:00, 1802.40it/s]
make buckets
number of images (including repeats) / 各bucketの画像枚数（繰り返し回数を含む）
bucket 0: resolution (512, 512), count: 300
mean ar error (without repeats): 0.0
preparing accelerator
Using accelerator 0.15.0 or above.
loading model for process 0/1
load StableDiffusion checkpoint: ./sd-models/Deliberate_v2.safetensors
loading u-net:
loading vae:
loading text encoder:
import network module: networks.lora
[Dataset 0]
caching latents.
100%|███████████████████████████████████████████| 30/30 [00:02<00:00, 12.80it/s]
create LoRA network. base dim (rank): 32, alpha: 32.0
neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
create LoRA for Text Encoder: 72 modules.
create LoRA for U-Net: 192 modules.
enable LoRA for text encoder
enable LoRA for U-Net
preparing optimizer, data loader etc.

===================================BUG REPORT===================================
Welcome to bitsandbytes. For bug reports, please submit your error trace to: https://github.com/TimDettmers/bitsandbytes/issues
For effortless bug reporting copy-paste your error into this form: https://docs.google.com/forms/d/e/1FAIpQLScPB8emS3Thkp66nvqwmjTEgxp8Y9ufuWTzFyr9kJ5AoI47dQ/viewform?usp=sf_link
================================================================================
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/usr/local/lib/python3.10/dist-packages/cv2/../../lib64'), PosixPath('/usr/local/nvidia/lib'), PosixPath('/usr/local/nvidia/lib64')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:105: UserWarning: /usr/local/lib/python3.10/dist-packages/cv2/../../lib64:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 did not contain libcudart.so as expected! Searching further paths...
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('//10.192.0.1'), PosixPath('443'), PosixPath('tcp')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('module'), PosixPath('//matplotlib_inline.backend_inline')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('tcp'), PosixPath('//10.192.7.18'), PosixPath('80')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('http'), PosixPath('//license-pai.cn-shanghai.data.aliyun.com')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('aigc-torch113-cu117-ubuntu22.04-v0.2.1'), PosixPath('dsw-registry-vpc.cn-shanghai.cr.aliyuncs.com/cloud-dsw/eas-service')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('tcp'), PosixPath('22'), PosixPath('//10.192.7.18')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('https'), PosixPath('//dsw-cn-shanghai.data.aliyun.com')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('/home/pai/bin/python')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('Asia/Shanghai')}
warn(
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cuda_setup/paths.py:27: UserWarning: WARNING: The following directories listed in your path were found to be non-existent: {PosixPath('lora-scripts/974a7713-7c05-4934-a28a-6071e8b5e150')}
warn(
CUDA_SETUP: WARNING! libcudart.so not found in any environmental path. Searching /usr/local/cuda/lib64...
WARNING: No libcudart.so found! Install CUDA or the cudatoolkit package (anaconda)!
CUDA SETUP: Loading binary /usr/local/lib/python3.10/dist-packages/bitsandbytes/libbitsandbytes_cpu.so...
/usr/local/lib/python3.10/dist-packages/bitsandbytes/cextension.py:48: UserWarning: The installed version of bitsandbytes was compiled without GPU support. 8-bit optimizers and GPU quantization are unavailable.
warn(
use 8-bit AdamW optimizer | {}
override steps. steps for 10 epochs is / 指定エポックまでのステップ数: 3000
running training / 学習開始
num train images * repeats / 学習画像の数×繰り返し回数: 300
num reg images / 正則化画像の数: 0
num batches per epoch / 1epochのバッチ数: 300
num epochs / epoch数: 10
batch size per device / バッチサイズ: 1
gradient accumulation steps / 勾配を合計するステップ数 = 1
total optimization steps / 学習ステップ数: 3000

The traceback enters sd-scripts/train_network.py at line 688 and then reaches bitsandbytes/functional.py at line 858 during optimizer.step. Start by tracing the optimizer configuration and the external call that raises the NameError. Done means the reported training run can pass the optimizer step without this exception.

Written by the indexing model from the issue text.