slow predictions with sam2 video predictor

I'm using the sam video predictor to create segmentations on 3d medical images, but I'm getting slow performance during propagate_in_video of ~1 fps. I'm using a g4dn.xlarge gpu, and the images are converted from dicom to jpg and stored in a temp folder before initializing the model state. I also set the forward and backward propagation to run in parallel.

I believe that I should be getting more than 10x the current performance, so any ideas why it's running so slowly?

"""
def process_prop(self, input_data: dict[str, any], state, box: np.ndarray, init_frame: int) -> list[dict]:
try:
predictions = []

frame_idx, object_ids, preds = self.model.add_new_points_or_box(
state, init_frame, 0, box=box[None, :]
)

# Create a prediction for the center slice
center_prediction = {
'mask': np.squeeze(preds.cpu().numpy() > 0).astype(np.uint8),
'pred_frame_id': init_frame
}
predictions.append(center_prediction)

# if making both forward and backward predictions, perform in parallel
if input_data['forward_prop'] is not None and input_data['backward_prop'] is not None:
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
# Deep copy the state for the backward propagation to avoid conflicts
backward_state = copy.deepcopy(state)

# Submit both propagation tasks
forward_future = executor.submit(
self.run_prop,
start_frame_idx=frame_idx + 1,
state=state,
reverse=False
)

backward_future = executor.submit(
self.run_prop,
start_frame_idx=frame_idx - 1,
state=backward_state,
reverse=True
)

# Collect results as they complete
forward_predictions = forward_future.result()
backward_predictions = backward_future.result()
predictions.extend(forward_predictions)
predictions.extend(backward_predictions)
else:
if input_data['forward_prop'] is not None:
forward_predictions = self.run_prop(
start_frame_idx=frame_idx + 1, state=state, reverse=False
)
predictions.extend(forward_predictions)

if input_data['backward_prop'] is not None:
backward_predictions = self.run_prop(
start_frame_idx=frame_idx - 1, state=state, reverse=True
)
predictions.extend(backward_predictions)

# Sort predictions by frame_id for consistent output
predictions.sort(key=lambda x: x['pred_frame_id'])
return predictions

except Exception as e:
logging.error(f"Error in prediction: {str(e)}")

def run_prop(self, start_frame_idx: int, state, reverse=False) -> list[dict[str, any]]:
"""
Run propagation and return predictions
"""
predictions = []

for frame_idx, object_ids, preds in self.model.propagate_in_video(
state, start_frame_idx=start_frame_idx, reverse=reverse
):
preds = np.array(preds.cpu() > 0, dtype=int).astype(np.uint8)
binary_mask = np.squeeze(preds)
predictions.append(
{
'mask': binary_mask,
'pred_frame_id': frame_idx
}
)

return predictions
"""

Start with the provided process_prop and run_prop entry points, focusing on add_new_points_or_box, propagate_in_video, state copying, and ThreadPoolExecutor behavior. Profile the forward and backward paths on the stated g4dn.xlarge setup and inspect the DICOM-to-JPG preparation; done means identifying the measured bottleneck and explaining the performance gap with a reproducible comparison.

Written by the indexing model from the issue text.