VideoReaderResize high RAM consumption

Hi! I build the pipeline to train multilabel video classification model:

class VideoPreprocessingPipeline(Pipeline):

    READER_NAME = 'VideoReaderResize'

    def __init__(
        self,

        # Pipeline args
        batch_size: int,
        num_threads: int,
        device_id: int,
        seed: int,
        prefetch_queue_depth: int,

        # Labels args
        labels_matrix: np.ndarray,

        # Video Reader Resize args
        video_filenames: List[str],
        sequence_length: int,
        random_shuffle: bool,
        initial_fill: int,
        step: int,
        stride: int,
        shard_id: int = 0,
        num_shards: int = 1,
        skip_vfr_check: bool = True,
        pad_last_batch: bool = False,
        resize_x: Optional[float] = None,
        resize_y: Optional[float] = None,
        resize_shorter: Optional[float] = None,
        resize_longer: Optional[float] = None,
        image_type: types.DALIImageType = types.DALIImageType.RGB,
        additional_decode_surfaces: int = 2,

        # Others
        rotation_angle_range: Tuple[float, float] = (-15, 15),
        mean: float = 0.,
        std: float = 1.,
        crop_w: int = 299,
        crop_h: int = 299,
        crop_pos_x_range: Tuple[float, float] = (0., 0.25),
        crop_pos_y_range: Tuple[float, float] = (0., 0.25),
        flip_probability: float = 0.5,
    ):
        super(VideoPreprocessingPipeline, self).__init__(
            batch_size,
            num_threads,
            device_id,
            seed=seed,
            prefetch_queue_depth=prefetch_queue_depth,
            exec_async=False,
            exec_pipelined=False
        )

        file_list = VideoPreprocessingPipeline.create_file_list(video_filenames)
        self.video_reader_resize = ops.VideoReaderResize(
            device="gpu",
            file_list=file_list,
            sequence_length=sequence_length,
            shard_id=shard_id,
            num_shards=num_shards,
            random_shuffle=random_shuffle,
            initial_fill=initial_fill,
            skip_vfr_check=skip_vfr_check,
            resize_x=resize_x,
            resize_y=resize_y,
            resize_shorter=resize_shorter,
            resize_longer=resize_longer,
            pad_last_batch=pad_last_batch,
            step=step,
            stride=stride,
            image_type=image_type,
            additional_decode_surfaces=additional_decode_surfaces
        )

        self.to_volumetric = ops.Reinterpret(
            device="gpu",
            layout="DHWC"
        )
        self.from_volumetric = ops.Reinterpret(
            device="gpu",
            layout="FHWC"
        )

        self.rotate = ops.Rotate(
            axis=[0, 0, 1],
            device="gpu",
            keep_size=True,
        )
        self.rotation_angle = ops.Uniform(range=rotation_angle_range)

        self.crop_mirror_normalize = ops.CropMirrorNormalize(
            device="gpu",
            mean=mean,
            std=std,
            crop_h=crop_h,
            crop_w=crop_w,
            output_layout="FCHW",
            out_of_bounds_policy='error'
        )
        self.crop_pos_x = ops.Uniform(range=crop_pos_x_range)
        self.crop_pos_y = ops.Uniform(range=crop_pos_y_range)
        self.do_flip = ops.CoinFlip(
            device="cpu",
            probability=flip_probability,
        )

        self.labels_lookup = VideoPreprocessingPipeline.get_labels_lookup_op(labels_matrix)

    @property
    def reader_name(self):
        return self.READER_NAME

    @staticmethod
    def get_labels_lookup_op(labels_matrix):

        def _get_labels(_index):
            index = cp.fromDlpack(_index)[0]
            index = cp.asnumpy(index)
            labels = cp.array(labels_matrix[index]).toDlpack()

            return labels

        op = ops.DLTensorPythonFunction(
            function=_get_labels,
            device='gpu',
            synchronize_stream=True,
            batch_processing=False
        )

        return op

    @staticmethod
    def create_file_list(video_filenames):
        tmp_file = NamedTemporaryFile(mode='w', delete=False)
        for i, filename in enumerate(video_filenames):
            tmp_file.write(f'{filename} {i}\n')
        tmp_file.flush()

        return tmp_file.name

    def define_graph(self):
        frames, label_indices, *_ = self.video_reader_resize(
            name=self.reader_name
        )

        frames = self.to_volumetric(frames)
        frames = self.rotate(frames, angle=self.rotation_angle())
        frames = self.from_volumetric(frames)

        frames = self.crop_mirror_normalize(
            frames,
            crop_pos_x=self.crop_pos_x(),
            crop_pos_y=self.crop_pos_y(),
            mirror=self.do_flip()
        )

        labels = self.labels_lookup(label_indices)

        return frames, labels

and wraps in with

train_iterator = DALIClassificationIterator(
    pipelines=[train_pipe],
    reader_name=train_pipe.reader_name,
    auto_reset=True,
    dynamic_shape=False,
    fill_last_batch=config.PAD_LAST_BATCH
)

My train dataset contains around 800k videos with various length and when I try to create only one preprocessing pipeline it consumes around 40Gb RAM which is affordable, but when I try to use multiple pipelines for multi-GPU training total RAM consumption is too high. What data VideoReader keeps and is it really necessary to use so much RAM? Is there any workaround for it?

Also, my second complaint is about initialization speed: for this amount of data it takes 20-30mins with almost no CPU load -- why it's so slow?

Start with the VideoReaderResize entry point and its interaction with DALIClassificationIterator; profile initialization and per-pipeline memory for the supplied 800k-video case. Done means identifying what retained data drives RAM and startup time, then documenting a reproducible workaround or scoped fix.