kohya-ss / kohya-ss/sd-scripts
so many errors I don't know where to start
- Dominant language
- Python
- Stars
- 7.2k
- Forks
- 1.2k
- Avg merge
- 11m
- Merged PRs (30d)
- 2
Description
stable diffusion webui works just fine, I've got automatic1111 and other forks all working just fine on this machine.
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ E:\kohya_ss\train_network.py:783 in │
│ │
│ 780 │ args = parser.parse_args() │
│ 781 │ args = train_util.read_config_from_file(args, parser) │
│ 782 │ │
│ ❱ 783 │ train(args) │
│ 784 │
│ │
│ E:\kohya_ss\train_network.py:157 in train │
│ │
│ 154 │ │ vae.requires_grad_(False) │
│ 155 │ │ vae.eval() │
│ 156 │ │ with torch.no_grad(): │
│ ❱ 157 │ │ │ train_dataset_group.cache_latents(vae, args.vae_batch_size, args.cache_laten │
│ 158 │ │ vae.to("cpu") │
│ 159 │ │ if torch.cuda.is_available(): │
│ 160 │ │ │ torch.cuda.empty_cache() │
│ │
│ E:\kohya_ss\library\train_util.py:1399 in cache_latents │
│ │
│ 1396 │ def cache_latents(self, vae, vae_batch_size=1, cache_to_disk=False, is_main_process= │
│ 1397 │ │ for i, dataset in enumerate(self.datasets): │
│ 1398 │ │ │ print(f"[Dataset {i}]") │
│ ❱ 1399 │ │ │ dataset.cache_latents(vae, vae_batch_size, cache_to_disk, is_main_process) │
│ 1400 │ │
│ 1401 │ def is_latent_cacheable(self) -> bool: │
│ 1402 │ │ return all([dataset.is_latent_cacheable() for dataset in self.datasets]) │
│ │
│ E:\kohya_ss\library\train_util.py:812 in cache_latents │
│ │
│ 809 │ │ │ img_tensors = torch.stack(images, dim=0) │
│ 810 │ │ │ img_tensors = img_tensors.to(device=vae.device, dtype=vae.dtype) │
│ 811 │ │ │ │
│ ❱ 812 │ │ │ latents = vae.encode(img_tensors).latent_dist.sample().to("cpu") │
│ 813 │ │ │ │
│ 814 │ │ │ for info, latent in zip(batch, latents): │
│ 815 │ │ │ │ if cache_to_disk: │
│ │
│ C:\Python310\lib\site-packages\diffusers\models\vae.py:566 in encode │
│ │
│ 563 │ │ self.use_slicing = False │
│ 564 │ │
│ 565 │ def encode(self, x: torch.FloatTensor, return_dict: bool = True) -> AutoencoderKLOut │
│ ❱ 566 │ │ h = self.encoder(x) │
│ 567 │ │ moments = self.quant_conv(h) │
│ 568 │ │ posterior = DiagonalGaussianDistribution(moments) │
│ 569 │
│ │
│ C:\Python310\lib\site-packages\torch\nn\modules\module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ C:\Python310\lib\site-packages\diffusers\models\vae.py:130 in forward │
│ │
│ 127 │ │
│ 128 │ def forward(self, x): │
│ 129 │ │ sample = x │
│ ❱ 130 │ │ sample = self.conv_in(sample) │
│ 131 │ │ │
│ 132 │ │ # down │
│ 133 │ │ for down_block in self.down_blocks: │
│ │
│ C:\Python310\lib\site-packages\torch\nn\modules\module.py:1501 in _call_impl │
│ │
│ 1498 │ │ if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks │
│ 1499 │ │ │ │ or _global_backward_pre_hooks or _global_backward_hooks │
│ 1500 │ │ │ │ or _global_forward_hooks or _global_forward_pre_hooks): │
│ ❱ 1501 │ │ │ return forward_call(*args, **kwargs) │
│ 1502 │ │ # Do not call functions when jit is used │
│ 1503 │ │ full_backward_hooks, non_full_backward_hooks = [], [] │
│ 1504 │ │ backward_pre_hooks = [] │
│ │
│ C:\Python310\lib\site-packages\torch\nn\modules\conv.py:463 in forward │
│ │
│ 460 │ │ │ │ │ │ self.padding, self.dilation, self.groups) │
│ 461 │ │
│ 462 │ def forward(self, input: Tensor) -> Tensor: │
│ ❱ 463 │ │ return self._conv_forward(input, self.weight, self.bias) │
│ 464 │
│ 465 class Conv3d(_ConvNd): │
│ 466 │ __doc__ = r"""Applies a 3D convolution over an input signal composed of several inpu │
│ │
│ C:\Python310\lib\site-packages\torch\nn\modules\conv.py:459 in _conv_forward │
│ │
│ 456 │ │ │ return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=sel │
│ 457 │ │ │ │ │ │ │ weight, bias, self.stride, │
│ 458 │ │ │ │ │ │ │ _pair(0), self.dilation, self.groups) │
│ ❱ 459 │ │ return F.conv2d(input, weight, bias, self.stride, │
│ 460 │ │ │ │ │ │ self.padding, self.dilation, self.groups) │
│ 461 │ │
│ 462 │ def forward(self, input: Tensor) -> Tensor: │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
RuntimeError: "slow_conv2d_cpu" not implemented for 'Half'
╭─────────────────────────────── Traceback (most recent call last) ────────────────────────────────╮
│ C:\Python310\lib\runpy.py:196 in _run_module_as_main │
│ │
│ 193 │ main_globals = sys.modules["__main__"].__dict__ │
│ 194 │ if alter_argv: │
│ 195 │ │ sys.argv[0] = mod_spec.origin │
│ ❱ 196 │ return _run_code(code, main_globals, None, │
│ 197 │ │ │ │ │ "__main__", mod_spec) │
│ 198 │
│ 199 def run_module(mod_name, init_globals=None, │
│ │
│ C:\Python310\lib\runpy.py:86 in _run_code │
│ │
│ 83 │ │ │ │ │ __loader__ = loader, │
│ 84 │ │ │ │ │ __package__ = pkg_name, │
│ 85 │ │ │ │ │ __spec__ = mod_spec) │
│ ❱ 86 │ exec(code, run_globals) │
│ 87 │ return run_globals │
│ 88 │
│ 89 def _run_module_code(code, init_globals=None, │
│ │
│ C:\Python310\Scripts\accelerate.exe\__main__.py:7 in │
│ │
│ [Errno 2] No such file or directory: 'C:\\Python310\\Scripts\\accelerate.exe\\__main__.py' │
│ │
│ C:\Python310\lib\site-packages\accelerate\commands\accelerate_cli.py:45 in main │
│ │
│ 42 │ │ exit(1) │
│ 43 │ │
│ 44 │ # Run │
│ ❱ 45 │ args.func(args) │
│ 46 │
│ 47 │
│ 48 if __name__ == "__main__": │
│ │
│ C:\Python310\lib\site-packages\accelerate\commands\launch.py:923 in launch_command │
│ │
│ 920 │ elif defaults is not None and defaults.compute_environment == ComputeEnvironment.AMA │
│ 921 │ │ sagemaker_launcher(defaults, args) │
│ 922 │ else: │
│ ❱ 923 │ │ simple_launcher(args) │
│ 924 │
│ 925 │
│ 926 def main(): │
│ │
│ C:\Python310\lib\site-packages\accelerate\commands\launch.py:579 in simple_launcher │
│ │
│ 576 │ process.wait() │
│ 577 │ if process.returncode != 0: │
│ 578 │ │ if not args.quiet: │
│ ❱ 579 │ │ │ raise subprocess.CalledProcessError(returncode=process.returncode, cmd=cmd) │
│ 580 │ │ else: │
│ 581 │ │ │ sys.exit(1) │
│ 582 │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
CalledProcessError: Command '['C:\\Python310\\python.exe', 'train_network.py', '--enable_bucket',
'--pretrained_model_name_or_path=E:/a1111/stable-diffusion-webui/models/Stable-diffusion/000/v1-5-pruned-emaonly.ckpt',
'--train_data_dir=E:/kohya_ss/train/PatriciaDoyle/images', '--resolution=768,768',
'--output_dir=E:/kohya_ss/train/PatriciaDoyle/model', '--logging_dir=E:/kohya_ss/train/PatriciaDoyle/log',
'--network_alpha=1', '--save_model_as=safetensors', '--network_module=networks.lora', '--text_encoder_lr=5e-05',
'--unet_lr=0.0001', '--network_dim=8', '--output_name=PatriciaDoyle', '--lr_scheduler_num_cycles=10',
'--learning_rate=0.0001', '--lr_scheduler=cosine', '--lr_warmup_steps=760', '--train_batch_size=1',
'--max_train_steps=7600', '--save_every_n_epochs=1', '--mixed_precision=fp16', '--save_precision=fp16',
'--cache_latents', '--optimizer_type=AdamW8bit', '--max_data_loader_n_workers=0', '--bucket_reso_steps=64',
'--xformers', '--bucket_no_upscale']' returned non-zero exit status 1.
Contributor guide
No contributing guide indexed for this repository
Research direction
Start with train_network.py:157 and library/train_util.py:812-1399, where latent caching reaches the VAE encoder and fails with the reported CPU Half convolution error. Reproduce the training failure, trace the dtype and device values through this path, and document or verify a correction that allows latent caching to complete.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- python, pytorch
- Domain
- machine-learning
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 15/100