How to achieve parallel upload of images Use TensorRT

Description

I'm trying to upload multiple images to the device in parallel. I'm first putting the images into page-locked memory and then using cudaMemcpyAsync from the host to the device. However, when I use NsightSystem, I find that my HtoD process is still serial. How can I achieve parallel image uploads? Is this related to the graphics card hardware? I also want to use the images uploaded in parallel for parallel EnqueueV3 inference. Is there anything I need to do?

Environment

TensorRT Version: 10.10

NVIDIA GPU: 40170tiSuper

NVIDIA Driver Version:572.16

CUDA Version:12.1

CUDNN Version:8.6

Relevant Files

1.Allocate memory：
allocated_src_size = GPUPreprocessConfig::getMaxImageBytes() * batch_size;
allocated_mid_size = target_width * target_height * 3 * batch_size;
max_batch_size = batch_size;

m_pinned_buffer_size = GPUPreprocessConfig::getMaxImageBytes() * batch_size;
cudaError_t ret = cudaHostAlloc(
(void**)&m_pinned_host_buffer,
m_pinned_buffer_size,
cudaHostAllocPortable | cudaHostAllocWriteCombined
);
cudaError_t err1 = cudaMalloc((void**)&d_src_data, allocated_src_size);
cudaError_t err2 = cudaMalloc((void**)&d_mid_data, allocated_mid_size);

2.uploadImagesToGPU
bool MultiStreamGPUProcessor::uploadImagesToGPU(const std::vectorcv::Mat& images)
{
const int total_batch_size = images.size();

  size_t host_offset = 0;
  for (int i = 0; i < total_batch_size; i++) {
      if (images[i].rows > 0 && images[i].cols > 0) {
          size_t img_bytes = images[i].rows * images[i].cols * 3;
          if (host_offset + img_bytes > m_pinned_buffer_size) {
              std::cerr << "Image too large for pinned buffer at index " << i << std::endl;
              return false;
          }
          memcpy(m_pinned_host_buffer + host_offset, images[i].data, img_bytes);
          host_offset += img_bytes;
      }
  }

  size_t src_gpu_offset = 0;
  size_t src_host_offset = 0;
  int images_processed = 0;

  for (int stream_idx = 0; stream_idx < MAX_UPLOAD_STREAMS; ++stream_idx) {
      int num_images_in_stream = total_batch_size / MAX_UPLOAD_STREAMS + (stream_idx < total_batch_size % MAX_UPLOAD_STREAMS ? 1 : 0);
      if (num_images_in_stream == 0) continue;

      size_t block_bytes = 0;
      for (int i = 0; i < num_images_in_stream; ++i) {
          int image_index = images_processed + i;
          if (images[image_index].rows > 0 && images[image_index].cols > 0) {
              block_bytes += images[image_index].rows * images[image_index].cols * 3;
          }
      }

      if (block_bytes > 0) {
          cudaError_t err = cudaMemcpyAsync(
              d_src_data + src_gpu_offset,
              m_pinned_host_buffer + src_host_offset,
              block_bytes,
              cudaMemcpyHostToDevice,
              m_upload_streams[stream_idx]
          );
          cudaEventRecord(m_transfer_events[stream_idx], m_upload_streams[stream_idx]);
      }

      src_gpu_offset += block_bytes;
      src_host_offset += block_bytes;
      images_processed += num_images_in_stream;
  }
  return true;

}

Start with the allocation code and MultiStreamGPUProcessor::uploadImagesToGPU, then inspect the cudaMemcpyAsync calls, upload streams, and transfer events using Nsight Systems. Check how the uploaded buffers are later used for EnqueueV3 inference. Done means establishing whether the current transfers and inference can run concurrently and documenting the required changes or hardware constraints.