Discussion: Batch size automatically

import torch

def test_batch_size(model, batch_size, input_shape):
    try:
        input_data = torch.rand((batch_size,) + input_shape).cuda()
        output_data = model(input_data)
        del input_data, output_data
        torch.cuda.empty_cache()
        return True
    except RuntimeError as e:
        if 'out of memory' in str(e):
            return False
        else:
            raise e

def find_max_batch_size(model, input_shape):
    if not torch.cuda.is_available():
        raise ValueError("No GPUs available.")

    min_batch_size = 1
    max_batch_size = 1024
    while min_batch_size < max_batch_size:
        mid_batch_size = (min_batch_size + max_batch_size) // 2
        if test_batch_size(model, mid_batch_size, input_shape):
            min_batch_size = mid_batch_size + 1
        else:
            max_batch_size = mid_batch_size
    return max_batch_size - 1

# Let's say we have a simple model:
model = torch.nn.Sequential(
    torch.nn.Linear(100, 64),
    torch.nn.ReLU(),
    torch.nn.Linear(64, 10)
).cuda()

max_batch_size = find_max_batch_size(model, input_shape=(100,))
print("Max batch size: ", max_batch_size)

This is a silly idea, that increments batch size. The idea is to change the random data with original datasets.
Another idea is taking the memory of gpu and calculate it

The issue contains a Python prototype for probing CUDA batch sizes, but names no repository files, tests, or entry points. First determine how TorchBench selects datasets and records GPU memory, then define the supported behavior and validation needed for automatic batch sizing; the work is done when those requirements are implemented and tested against representative original datasets.