TensorRT produces wrong results since the CumSum operator produces identical outputs for different 'reverse' attribute

Description

For the following simple onnx model,

the only difference for the two CumSum operators is the 'reverse' attribute. 'output1' and 'output2' should be different. However, when I run it using tensorrt, 'output1' and 'output2' are identical.
The results produced by tensorrt are as follows:

TensorRT: 
 [array([[[-1.3519955 , -0.7268443 , -0.1298989 , -0.26899588,
          0.00713499],
        [ 0.02994417,  0.09074355,  0.01346359, -0.08502126,
          1.2606536 ],
        [-2.9652479 , -0.7773497 , -0.7120025 , -2.4827871 ,
         -1.1241275 ],
        [ 5.4411387 ,  4.2780757 ,  2.7492511 ,  1.1518208 ,
          1.2441877 ]],

       [[ 0.6803112 ,  1.1280711 ,  0.92128164,  0.65729713,
          1.0393754 ],
        [ 4.3415256 ,  2.2513127 ,  2.7186024 ,  0.8549059 ,
         -0.38409668],
        [-3.9618876 , -3.6651313 , -1.5512493 , -0.29409987,
         -0.48622572],
        [ 0.45760983, -0.92412144, -1.1371686 , -1.261433  ,
         -1.4493794 ]],

       [[-1.8433855 , -2.2286081 , -2.786633  , -1.7782474 ,
         -1.9025584 ],
        [ 1.0052809 ,  2.1252763 ,  3.09984   ,  2.5171452 ,
          3.0486135 ],
        [ 2.4097364 ,  2.1245673 ,  0.9027792 ,  0.77380884,
          0.20755884],
        [ 1.6885381 ,  0.859954  ,  1.7572367 ,  2.2161024 ,
          1.3712262 ]]], dtype=float32), 
array([[[-1.3519955 , -0.7268443 , -0.1298989 , -0.26899588,
          0.00713499],
        [ 0.02994417,  0.09074355,  0.01346359, -0.08502126,
          1.2606536 ],
        [-2.9652479 , -0.7773497 , -0.7120025 , -2.4827871 ,
         -1.1241275 ],
        [ 5.4411387 ,  4.2780757 ,  2.7492511 ,  1.1518208 ,
          1.2441877 ]],

       [[ 0.6803112 ,  1.1280711 ,  0.92128164,  0.65729713,
          1.0393754 ],
        [ 4.3415256 ,  2.2513127 ,  2.7186024 ,  0.8549059 ,
         -0.38409668],
        [-3.9618876 , -3.6651313 , -1.5512493 , -0.29409987,
         -0.48622572],
        [ 0.45760983, -0.92412144, -1.1371686 , -1.261433  ,
         -1.4493794 ]],

       [[-1.8433855 , -2.2286081 , -2.786633  , -1.7782474 ,
         -1.9025584 ],
        [ 1.0052809 ,  2.1252763 ,  3.09984   ,  2.5171452 ,
          3.0486135 ],
        [ 2.4097364 ,  2.1245673 ,  0.9027792 ,  0.77380884,
          0.20755884],
        [ 1.6885381 ,  0.859954  ,  1.7572367 ,  2.2161024 ,
          1.3712262 ]]], dtype=float32)]

This model can also be executed by onnxruntime, the results are as follows:

ONNXRuntime:
 [array([[[-0.6251511 , -1.2220965 , -1.08299953, -1.35913041,
         -1.35199542],
        [-0.06079938,  0.01648058,  0.11496543, -1.23070944,
          0.02994417],
        [-2.18789816, -2.25324535, -0.48246073, -1.84112047,
         -2.96524798],
        [ 1.16306293,  2.69188762,  4.28931797,  4.19695108,
          5.44113879]],

       [[-0.4477599 , -0.24097046,  0.02301404, -0.35906422,
          0.6803112 ],
        [ 2.09021282,  1.62292314,  3.48661971,  4.7256223 ,
          4.34152561],
        [-0.29675618, -2.41063824, -3.66778758, -3.47566174,
         -3.96188746],
        [ 1.38173127,  1.59477848,  1.71904282,  1.90698928,
          0.45760983]],

       [[ 0.38522267,  0.94324744, -0.0651381 ,  0.05917305,
         -1.8433854 ],
        [-1.11999547, -2.09455901, -1.5118643 , -2.0433327 ,
          1.00528085],
        [ 0.28516909,  1.50695714,  1.63592756,  2.20217758,
          2.40973642],
        [ 0.82858407, -0.06869864, -0.52756426,  0.31731185,
          1.68853804]]]), 
array([[[-1.35199542, -0.72684432, -0.12989892, -0.26899589,
          0.00713499],
        [ 0.02994417,  0.09074355,  0.01346359, -0.08502126,
          1.26065361],
        [-2.96524798, -0.77734982, -0.71200264, -2.48278725,
         -1.12412751],
        [ 5.44113879,  4.27807586,  2.74925117,  1.15182082,
          1.24418771]],

       [[ 0.6803112 ,  1.1280711 ,  0.92128167,  0.65729716,
          1.03937542],
        [ 4.34152561,  2.25131279,  2.71860248,  0.8549059 ,
         -0.38409668],
        [-3.96188746, -3.66513129, -1.55124922, -0.29409988,
         -0.48622572],
        [ 0.45760983, -0.92412144, -1.13716865, -1.26143299,
         -1.44937944]],

       [[-1.8433854 , -2.22860807, -2.78663284, -1.7782473 ,
         -1.90255845],
        [ 1.00528085,  2.12527633,  3.09983987,  2.51714516,
          3.04861355],
        [ 2.40973642,  2.12456733,  0.90277928,  0.77380887,
          0.20755884],
        [ 1.68853804,  0.85995397,  1.75723669,  2.2161023 ,
          1.37122619]]])]

From the above results, we can see that the first results of tensorrt and onnxruntime are different, 93.3% elements are mismatched.

Mismatched elements: 56 / 60 (93.3%)
Max absolute difference among violations: 4.7256223
Max relative difference among violations: 41.78038425

Environment

TensorRT Version: 10.12.0.36

NVIDIA GPU: GeForce RTX 3080

NVIDIA Driver Version: 535.183.01

CUDA Version: 12.2

CUDNN Version: none

Operating System: ubuntu 20.04

Python Version (if applicable): 3.12.9

Steps To Reproduce

This issue can be reproduced by the following code with the model in the attachment.

from typing import Dict, List, Literal, Optional
import sys
import os

import numpy as np
import onnx
import onnxruntime

import tensorrt as trt
import pycuda.driver as cuda
import pycuda.autoinit

import argparse
import pickle


def test():
    onnx_model = onnx.load('333.onnx')

    with open("inputs.pkl", "rb") as fp:
        inputs = pickle.load(fp)

    try:
        ort_session = onnxruntime.InferenceSession(
            onnx_model.SerializeToString(), providers=["CPUExecutionProvider"]
        )
        ort_output = ort_session.run([], inputs)
    except Exception as e:
        print(e)
        print("This model cannot be executed by onnxruntime!")
        sys.exit(1)
    
    print("ONNXRuntime:\n", ort_output)
    
    #--------------------------------------------------------
        
    trt_logger = trt.Logger(trt.Logger.WARNING)
    trt.init_libnvinfer_plugins(trt_logger, '')
    builder = trt.Builder(trt_logger)
    network = builder.create_network(flags=1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))

    parser = trt.OnnxParser(network, trt_logger)
    with open('333.onnx', 'rb') as model_file:
        if not parser.parse(model_file.read()):
            for error in range(parser.num_errors):
                print(parser.get_error(error))
            sys.exit(1)
    
    config = builder.create_builder_config()
    serialized_engine = builder.build_serialized_network(network, config)
    
    if serialized_engine == None:
        sys.exit(1)
    
    with open("engine.trt", "wb") as f:
        f.write(serialized_engine)
        
    with open("engine.trt", "rb") as f, trt.Runtime(trt_logger) as runtime:
        engine = runtime.deserialize_cuda_engine(f.read())
        
    context = engine.create_execution_context()

    inputs_trt, outputs_trt, bindings = [], [], []
    stream = cuda.Stream()
    input_name = []
    output_shape_dtype = []
    #------------------------------------------------------------
    for binding in engine:
        size = trt.volume(engine.get_tensor_shape(binding))
        dtype = trt.nptype(engine.get_tensor_dtype(binding))
        host_mem = cuda.pagelocked_empty(size, dtype)
        device_mem = cuda.mem_alloc(host_mem.nbytes)
        bindings.append({'name':binding, 'address':int(device_mem)})
        
        if engine.get_tensor_mode(binding) == trt.TensorIOMode.INPUT:
            inputs_trt.append({'host': host_mem, 'device': device_mem})
            input_name.append(binding)
        else:
            outputs_trt.append({'host': host_mem, 'device': device_mem})
            output_shape = engine.get_tensor_shape(binding)
            output_shape_dtype.append({'shape':output_shape, 'dtype':dtype})

    for i, input_mem in enumerate(inputs_trt):
        inp = np.ravel(inputs[input_name[i]])
        np.copyto(input_mem['host'], inp)
        cuda.memcpy_htod_async(input_mem['device'], input_mem['host'], stream)

    for bind in bindings:
        name = bind['name']
        addr = bind['address']
        context.set_tensor_address(name, addr)
    
    context.execute_async_v3(stream_handle=stream.handle)
    
    trt_output = []
    for i, output_mem in enumerate(outputs_trt):
        cuda.memcpy_dtoh_async(output_mem['host'], output_mem['device'], stream)
        out_shape = output_shape_dtype[i]['shape']
        out = output_mem['host'].reshape(out_shape)
        trt_output.append(out)

    stream.synchronize()
    
    print("TensorRT: \n", trt_output)
    
    assert len(ort_output) == len(trt_output), "Unequal number of outputs"
    
    np.testing.assert_allclose(trt_output[1], ort_output[1], rtol=0.1, atol=0.1) # OK
    np.testing.assert_allclose(trt_output[0], ort_output[0], rtol=0.1, atol=0.1) # BAD
        

    
if __name__ == "__main__":
    test()
    

testcase.zip

Commands or scripts:

Have you tried the latest release?: yes

Can this model run on other frameworks? For example run ONNX model with ONNXRuntime (polygraphy run <model.onnx> --onnxrt): the mode can be executed by onnxruntime.