Video reader of full sequence at given sampling rate

Describe the question.

Hi,

I would like to use DALI to extract frames at a rate of 2 FPS, while the original videos are encoded at 25 FPS. Note, that the length of the videos varies.

I think that I can use the keyword stride to sample one every N frames, but I don't know how to get rid of sequence_length in a proper way.

The basic idea is to reproduce the same output as I would have with this kind of code:

import numpy as np
from tqdm import tqdm
import cv2
import moviepy.editor

def getDuration(video_path):
    """Get the duration (in seconds) for a video.

    Keyword arguments:
    video_path -- the path of the video
    """
    return moviepy.editor.VideoFileClip(video_path).duration

class FrameCV():
    def __init__(self, video_path, FPS=2, start=None, duration=None):
        """Create a list of frame from a video using OpenCV.

        Keyword arguments:
        video_path -- the path of the video
        FPS -- the desired FPS for the frames (default:2)
        transform -- the desired transformation for the frames (default:2)
        start -- the desired starting time for the list of frames (default:None)
        duration -- the desired duration time for the list of frames (default:None)
        """

        self.FPS = FPS
        self.transform = transform
        self.start = start
        self.duration = duration

        # read video
        vidcap = cv2.VideoCapture(video_path)
        # read FPS
        self.fps_video = vidcap.get(cv2.CAP_PROP_FPS)
        # read duration
        self.time_second = getDuration(video_path)        

        # loop until the number of frame is consistent with the expected number of frame, 
        # given the duratio nand the FPS
        good_number_of_frames = False
        while not good_number_of_frames: 

            # read video
            vidcap = cv2.VideoCapture(video_path)
            
            # get number of frames
            self.numframe = int(self.time_second*self.fps_video)
            
            # frame drop ratio
            drop_extra_frames = self.fps_video/self.FPS

            # init list of frames
            self.frames = []

            # TQDM progress bar
            pbar = tqdm(range(self.numframe), desc='Grabbing Video Frames', unit='frame')
            i_frame = 0
            ret, frame = vidcap.read()

            # loop until no frame anymore
            while ret:
                # update TQDM
                pbar.update(1)
                i_frame += 1
                
                # skip until starting time
                if self.start is not None:
                    if i_frame < self.fps_video * self.start:
                        ret, frame = vidcap.read()
                        continue

                # skip after duration time
                if self.duration is not None:
                    if i_frame > self.fps_video * (self.start + self.duration):
                        ret, frame = vidcap.read()
                        continue
                        

                if (i_frame % drop_extra_frames < 1):
                    # append the frame to the list
                    self.frames.append(frame)
                
                # read next frame
                ret, frame = vidcap.read()

            # check if the expected number of frames were read
            if self.numframe - (i_frame+1) <=1:
                logging.debug("Video read properly")
                good_number_of_frames = True
            else:
                logging.debug("Video NOT read properly, adjusting fps and read again")
                self.fps_video = (i_frame+1) / self.time_second

        # convert frame from list to numpy array
        self.frames = np.array(self.frames)

    def __len__(self):
        """Return number of frames."""
        return len(self.frames)

    def __iter__(self, index):
        """Return frame at given index."""
        return self.frames[index]

I also met an error while I wanted to return the frame number output using enable_frame_num=True. In the documentation, it is specified that filenames must be passed, which is the case in my test. However, I have the following error:

Error when constructing operator: readers__Video encountered:
[/opt/dali/dali/operators/reader/video_reader_op.h:78] Assert on "can_use_frames_timestamps_ || !enable_frame_num_" failed: frame numbers can be enabled only when `file_list`, or `filenames` with `labels` argument are passed

I understand the error, but I think that the documentation is misleading as it is not mentioned that labels must be passed (I don't specify any labels in my example).

The code I used is directly derived from one of your example:

@pipeline_def
def create_video_reader_pipeline(sequence_length, files, crop_size, stride=1):
    images, num_frames = fn.readers.video(device="gpu", filenames=files, sequence_length=sequence_length,
                              normalized=False, random_shuffle=False, image_type=types.RGB,
                              dtype=types.UINT8, initial_fill=16, pad_last_batch=True, name="Reader",
                              stride=stride, enable_frame_num=True,
                             )
    images = fn.crop(images, crop=crop_size, dtype=types.FLOAT,
                     crop_pos_x=fn.random.uniform(range=(0.0, 1.0)),
                     crop_pos_y=fn.random.uniform(range=(0.0, 1.0)))

    images = fn.transpose(images, perm=[3, 0, 1, 2])

    return images, num_frames

class DALILoader():
    def __init__(self, batch_size, file_root, sequence_length, crop_size, stride=1):
        container_files = [os.path.join(root, f) for root, _, files in os.walk(file_root) for f in files if "mkv" in f]
        self.pipeline = create_video_reader_pipeline(batch_size=batch_size,
                                                     sequence_length=sequence_length,
                                                     num_threads=2,
                                                     device_id=0,
                                                     files=container_files,
                                                     crop_size=crop_size,
                                                     stride=stride,
                                                    )
        self.pipeline.build()
        self.epoch_size = self.pipeline.epoch_size("Reader")
        self.dali_iterator = pytorch.DALIGenericIterator(self.pipeline,
                                                         ["data"],
                                                         reader_name="Reader",
                                                         last_batch_policy=pytorch.LastBatchPolicy.PARTIAL,
                                                         auto_reset=True)

    def __len__(self):
        return int(self.epoch_size)

    def __iter__(self):
        return self.dali_iterator.__iter__()

loader = DALILoader(2, "path/to/data", 10, [224, 398], 13)

Thanks in advance,

Renaud

Check for duplicates

• I have searched the open bugs/issues and have found no duplicates for this bug report