Training on 2x H100 on Ubuntu and speed is same as 1x H100 what we are doing wrong?

When training batch size 4 on H100 the speed is 1.27 second / it

When training batch size 4 on 2x H100 the speed is 2.05 second / it

So basically we almost got no speed boost from multiple GPU training

Is this expected? I am training on SDXL RealVis XL model with 1024 no bucketing

We are using latest bmaltais Kohya GUI on Ubuntu with the below multi-gpu configuration


@kohya-ss @bmaltais

this below is training json config

```
{
"adaptive_noise_scale": 0,
"additional_parameters": "--max_grad_norm=0.0 --no_half_vae --train_text_encoder",
"async_upload": false,
"bucket_no_upscale": true,
"bucket_reso_steps": 64,
"cache_latents": true,
"cache_latents_to_disk": true,
"caption_dropout_every_n_epochs": 0,
"caption_dropout_rate": 0,
"caption_extension": "",
"clip_skip": 1,
"color_aug": false,
"dataset_config": "",
"debiased_estimation_loss": false,
"dynamo_backend": "no",
"dynamo_mode": "default",
"dynamo_use_dynamic": false,
"dynamo_use_fullgraph": false,
"enable_bucket": false,
"epoch": 50,
"extra_accelerate_launch_args": "",
"flip_aug": false,
"full_bf16": true,
"full_fp16": false,
"gpu_ids": "1,2",
"gradient_accumulation_steps": 1,
"gradient_checkpointing": false,
"huber_c": 0.1,
"huber_schedule": "snr",
"huggingface_path_in_repo": "",
"huggingface_repo_id": "",
"huggingface_repo_type": "",
"huggingface_repo_visibility": "",
"huggingface_token": "",
"ip_noise_gamma": 0,
"ip_noise_gamma_random_strength": false,
"keep_tokens": 0,
"learning_rate": 8e-06,
"learning_rate_te": 1e-05,
"learning_rate_te1": 3e-06,
"learning_rate_te2": 0,
"log_tracker_config": "",
"log_tracker_name": "",
"log_with": "",
"logging_dir": "",
"loss_type": "l2",
"lr_scheduler": "constant",
"lr_scheduler_args": "",
"lr_scheduler_num_cycles": 1,
"lr_scheduler_power": 1,
"lr_warmup": 0,
"main_process_port": 0,
"masked_loss": false,
"max_bucket_reso": 2048,
"max_data_loader_n_workers": 0,
"max_resolution": "1024,1024",
"max_timestep": 1000,
"max_token_length": 75,
"max_train_epochs": 0,
"max_train_steps": 0,
"mem_eff_attn": false,
"metadata_author": "",
"metadata_description": "",
"metadata_license": "",
"metadata_tags": "",
"metadata_title": "",
"min_bucket_reso": 256,
"min_snr_gamma": 0,
"min_timestep": 0,
"mixed_precision": "bf16",
"model_list": "custom",
"multi_gpu": true,
"multires_noise_discount": 0,
"multires_noise_iterations": 0,
"no_token_padding": false,
"noise_offset": 0,
"noise_offset_random_strength": false,
"noise_offset_type": "Original",
"num_cpu_threads_per_process": 4,
"num_machines": 1,
"num_processes": 2,
"optimizer": "Adafactor",
"optimizer_args": "scale_parameter=False relative_step=False warmup_init=False weight_decay=0.01",
"output_dir": "/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion",
"output_name": "shoes_test_2",
"persistent_data_loader_workers": false,
"pretrained_model_name_or_path": "/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion/RealVisXL_V4.0.safetensors",
"prior_loss_weight": 1,
"random_crop": false,
"reg_data_dir": "",
"resume": "",
"resume_from_huggingface": "",
"sample_every_n_epochs": 0,
"sample_every_n_steps": 0,
"sample_prompts": "",
"sample_sampler": "euler_a",
"save_every_n_epochs": 10,
"save_every_n_steps": 0,
"save_last_n_steps": 0,
"save_last_n_steps_state": 0,
"save_model_as": "safetensors",
"save_precision": "bf16",
"save_state": false,
"save_state_on_train_end": false,
"save_state_to_huggingface": false,
"scale_v_pred_loss_like_noise_pred": false,
"sdxl": true,
"seed": 0,
"shuffle_caption": false,
"stop_text_encoder_training": 0,
"train_batch_size": 4,
"train_data_dir": "/home/Ubuntu/Desktop/shoes_train_datasets/test1/img",
"v2": false,
"v_parameterization": false,
"v_pred_like_loss": 0,
"vae": "stabilityai/sdxl-vae",
"vae_batch_size": 8,
"wandb_api_key": "",
"wandb_run_name": "",
"weighted_captions": false,
"xformers": "xformers"
}
```

TOML file

```
bucket_no_upscale = true
bucket_reso_steps = 64
cache_latents = true
cache_latents_to_disk = true
clip_skip = 1
dynamo_backend = "no"
epoch = 50
full_bf16 = true
gradient_accumulation_steps = 1
huber_c = 0.1
huber_schedule = "snr"
learning_rate = 8e-6
learning_rate_te1 = 3e-6
loss_type = "l2"
lr_scheduler = "constant"
lr_scheduler_args = []
lr_scheduler_num_cycles = 1
lr_scheduler_power = 1
max_bucket_reso = 2048
max_data_loader_n_workers = 0
max_timestep = 1000
max_token_length = 75
max_train_steps = 1175
min_bucket_reso = 256
mixed_precision = "bf16"
noise_offset_type = "Original"
optimizer_args = [ "scale_parameter=False", "relative_step=False", "warmup_init=False", "weight_decay=0.01",]
optimizer_type = "Adafactor"
output_dir = "/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion"
output_name = "shoes_test_2"
pretrained_model_name_or_path = "/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion/RealVisXL_V4.0.safetensors"
prior_loss_weight = 1
resolution = "1024,1024"
sample_prompts = "/home/Ubuntu/apps/stable-diffusion-webui/models/Stable-diffusion/prompt.txt"
sample_sampler = "euler_a"
save_every_n_epochs = 10
save_model_as = "safetensors"
save_precision = "bf16"
train_batch_size = 4
train_data_dir = "/home/Ubuntu/Desktop/shoes_train_datasets/test1/img"
vae = "stabilityai/sdxl-vae"
vae_batch_size = 8
xformers = true
```

Start with the multi-GPU settings in the supplied JSON and TOML configurations, especially multi_gpu, gpu_ids, num_processes, and train_batch_size. Reproduce the one-GPU and two-GPU timings, then inspect the training launch path to determine whether the observed scaling is expected and document the relevant bottleneck or configuration issue.

Written by the indexing model from the issue text.