GPU Utilization drops gradually over time using Executor API

System Info

• CPU architecture: x86_64

• CPU/Host memory size: 256GB

• GPU properties

  • GPU name: NVIDIA H100, NVIDIA L40S

• Libraries

  • TensorRT-LLM branch or tag: v0.18.0.dev2025020400

• NVIDIA driver version: 565.57.01

• OS: Debian

Who can help?

@kaiyux @byshiue

Information

• The official example scripts
• My own modified scripts

Tasks

• An officially supported task in the examples folder (such as GLUE/SQuAD, ...)
• My own task or dataset (give details below)

Reproduction

1. Build whisper engine using the official example
2. Run the following script:

import argparse
import json
import os
import time

import torch
from datasets import load_dataset

os.environ["TLLM_LOG_LEVEL"] = "ERROR"
import tensorrt_llm.bindings.executor as trtllm  # noqa: E402
from tqdm import tqdm  # noqa: E402
from whisper import log_mel_spectrogram, pad_or_trim  # noqa: E402
from whisper.tokenizer import get_tokenizer  # noqa: E402


def load_earnings22():
    bad_transcripts = [
        "<crosstalk>.",
        "Um- <inaudible>.",
        "Um, <inaudible>.",
        "<unk>.",
        "<silence>.",
        "Uh.",
        "mm-hmm <affirmative>.",
        "Um.",
        "<laugh>.",
        "Hmm.",
        "<inaudible>.",
    ]
    dataset = load_dataset("distil-whisper/earnings22", "chunked")["test"]
    dataset = dataset.remove_columns(["file_id", "segment_id", "start_ts", "end_ts"])
    dataset = dataset.filter(
        lambda row: all(
            row["transcription"] != bad_transcript for bad_transcript in bad_transcripts
        )
        and 160 <= row["audio"]["array"].shape[0] <= 480000
    )
    audios, references = list(
        zip(
            *[
                (torch.from_numpy(row["audio"]["array"]).float(), row["transcription"])
                for row in dataset
            ]
        )
    )

    return audios, references


def load_model(engine_path: str, device: int):
    num_beams = 5
    executor = trtllm.Executor(
        encoder_model_path=os.path.join(engine_path, "encoder"),
        decoder_model_path=os.path.join(engine_path, "decoder"),
        model_type=trtllm.ModelType.ENCODER_DECODER,
        executor_config=trtllm.ExecutorConfig(
            num_beams,
            max_batch_size=96,
            kv_cache_config=trtllm.KvCacheConfig(
                free_gpu_memory_fraction=0.90,
                cross_kv_cache_fraction=0.4,
                enable_block_reuse=False,
            ),
            parallel_config=trtllm.ParallelConfig(device_ids=[device]),
        ),
    )

    with open(os.path.join(engine_path, "encoder", "config.json"), "r") as f:
        encoder_config = json.load(f)
    n_vocab = encoder_config["pretrained_config"]["vocab_size"]
    n_mels = encoder_config["pretrained_config"]["n_mels"]
    is_multilingual = n_vocab >= 51865

    num_languages = n_vocab - 51765 - int(is_multilingual)
    tokenizer = get_tokenizer(multilingual=is_multilingual, num_languages=num_languages)

    return executor, tokenizer, n_mels


def inference(model, tokenizer, n_mels, audios):
    features = [
        pad_or_trim(
            log_mel_spectrogram(audio, n_mels=n_mels, padding=160)[:, :-1], 3000
        )
        for audio in tqdm(audios, desc="Extracting features")
    ]
    features = features * 10

    prompt = tokenizer.sot_sequence_including_notimestamps

    requests = [
        trtllm.Request(
            input_token_ids=prompt,
            max_tokens=210,
            encoder_input_features=feature.T.half().contiguous(),  # mel features
            encoder_output_length=feature.shape[1] // 2,
            end_id=tokenizer.eot,
            pad_id=tokenizer.eot,
            sampling_config=trtllm.SamplingConfig(beam_width=5),
            output_config=trtllm.OutputConfig(
                return_context_logits=False,
                return_log_probs=True,
                return_generation_logits=False,
                return_encoder_output=False,
                exclude_input_from_output=True,
            ),
        )
        for feature in features
    ]
    start = time.time()
    request_ids = model.enqueue_requests(requests)

    pbar = tqdm(total=len(request_ids), desc="Transcribing...")
    while model.get_num_responses_ready() < len(request_ids):
        pbar.update(model.get_num_responses_ready() - pbar.n)
        time.sleep(0.1)
    pbar.update(model.get_num_responses_ready() - pbar.n)
    pbar.close()

    results = model.await_responses(request_ids)
    duration = time.time() - start
    best_beams = [
        result[0].result.cum_log_probs.index(max(result[0].result.cum_log_probs))
        for result in results
    ]
    output_sequences = [
        result[0].result.output_token_ids[beam]
        for result, beam in zip(results, best_beams)
    ]
    transcriptions = [
        tokenizer.decode(output_sequence) for output_sequence in output_sequences
    ]
    token_counts = [len(output_sequence) + 1 for output_sequence in output_sequences]
    return transcriptions, duration, token_counts


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="WER benchmark")

    parser.add_argument("--model_path", type=str)

    parser.add_argument("--device", type=int, default=0)

    args = parser.parse_args()

    model, tokenizer, n_mels = load_model(args.model_path, args.device)

    audios, references = load_earnings22()

    transcriptions, duration, token_counts = inference(model, tokenizer, n_mels, audios)

    print(f"Transcription Speed: {(sum(token_counts) / duration):.2f} toks/s")

Expected behavior

GPU Utilization should stay constant as long as there is no power or temperature constraints and throttling and there are enough requests to saturate the engine

actual behavior

GPU and Memory utilization drops gradually over time, this can be verified by collecting the data using nvidia-smi pmon during the test, this is also reflected on inference speed, since the dataset is replicated 10 times, the inference speed should be the same but it shows a decline of around 20% and this decline increases if the test runs for even more time.
This issue was reproduced using versions from 0.15.0.dev up to the current main branch on 4 GPUs, 2xH100 and 2xL40S, a simple matmul stress test was run for 2 hours and the utilization stayed at 100% indicating that there are no hardware issues.
This graph shows the GPU util on H100 when the test is run 4 times using different TRT-LLM versions:

This is on L40S:

additional notes

I can profile the process using nsys if needed, I will also try to upload the request stats and iteration stats for this test

Start with the provided Python reproduction using the TensorRT-LLM Executor API and monitor it with nvidia-smi pmon; compare repeated runs across the reported versions and GPUs. Use nsys profiling if needed to locate the source of the gradual utilization and inference-speed decline. Done means sustained utilization and stable inference speed under the repeated request workload.