real life metric measurements with Ruicheng/moge-2-vitl-normal

Hi

Getting the point cloud result seems to be larger (2x? more? less?) than real-life distances.

What am I doing wrong?

Thank you!

"""
Google Colab script for video to point cloud conversion using MoGe

Cell 1 - Install dependencies:
!pip install torch torchvision git+https://github.com/microsoft/MoGe.git opencv-contrib-python open3d

Cell 2 - Main script:
"""

# Imports
import cv2
import numpy as np
import open3d as o3d
import torch
from moge.model.v2 import MoGeModel
from google.colab import files
import time
import os
import shutil
import threading
from queue import Queue
from dataclasses import dataclass
from typing import Tuple
import csv

@dataclass
class FrameData:
    """Container for frame data through the pipeline"""
    index: int
    original: np.ndarray
    processed: np.ndarray = None
    tensor: torch.Tensor = None

# Load model
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Using device: {device}")
model = MoGeModel.from_pretrained("Ruicheng/moge-2-vitl-normal").to(device).half().eval()

# Create/clean output folder
output_folder = 'point_clouds'
if os.path.exists(output_folder):
    shutil.rmtree(output_folder)
os.makedirs(output_folder)

# Open CSV file for intrinsics
intrinsics_csv = open('camera_intrinsics.csv', 'w', newline='')
intrinsics_writer = csv.writer(intrinsics_csv)
intrinsics_writer.writerow(['frame_index', 'fx', 'fy', 'cx', 'cy', 'fov_x_deg', 'fov_y_deg'])

# Video
video_path = 'Joe.mp4'

# Parameters
frame_interval_sec = 0.5  # Interval between frames to process (seconds)
scale_percent = 50        # Percentage to downscale frames
min_distance = 1.5       # Minimum distance from camera (meters)
fov_x_degrees = 81.0     # Horizontal field of view in degrees

# Process video
cap = cv2.VideoCapture(video_path)
fps = cap.get(cv2.CAP_PROP_FPS)
frame_skip = int(fps * frame_interval_sec)
width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))


# Set up queues
gpu_queue = Queue(maxsize=8)
write_queue = Queue()

def preprocess_frame(frame_data: FrameData, scale_percent: int) -> FrameData:
    """Preprocess a single frame: resize and convert to tensor"""
    frame = frame_data.original
    
    # Downscale if needed
    if scale_percent != 100:
        scale = scale_percent / 100
        frame = cv2.resize(frame, (int(width*scale), int(height*scale)))
    
    # Convert BGR to RGB
    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    
    # Create tensor
    tensor = torch.from_numpy(rgb/255.0).to(device, dtype=torch.float16).permute(2,0,1)
    
    frame_data.processed = rgb
    frame_data.tensor = tensor
    return frame_data

def filter_points_gpu(points_gpu: torch.Tensor, mask_gpu: torch.Tensor, rgb_tensor: torch.Tensor, 
                     min_distance: float = 1.5) -> Tuple[np.ndarray, np.ndarray]:
    """Filter point cloud data on GPU before transferring to CPU"""
    # Calculate Euclidean distance from camera
    distance = torch.sqrt(points_gpu[..., 0]**2 + points_gpu[..., 1]**2 + points_gpu[..., 2]**2)
    
    # Filter points based on:
    # 1. MoGe's confidence mask
    # 2. Minimum distance from camera
    # 3. Valid (finite) values
    valid_mask = mask_gpu & (distance > min_distance) & torch.isfinite(points_gpu).all(dim=-1)
    
    # Flatten and extract valid points on GPU
    h, w = mask_gpu.shape
    points_flat = points_gpu.view(-1, 3)
    rgb_flat = rgb_tensor.view(-1, 3)
    mask_flat = valid_mask.view(-1)
    
    valid_points_gpu = points_flat[mask_flat]
    valid_colors_gpu = rgb_flat[mask_flat]
    
    # Transfer only filtered points to CPU
    valid_points = valid_points_gpu.cpu().numpy()
    valid_colors = valid_colors_gpu.cpu().numpy()
    
    return valid_points, valid_colors

def writer_thread():
    """Background thread for writing PLY files"""
    while True:
        item = write_queue.get()
        if item is None:
            break
        
        filename, points, colors = item
        try:
            # Create Open3D object here (in background thread)
            pcd = o3d.geometry.PointCloud()
            pcd.points = o3d.utility.Vector3dVector(points)
            pcd.colors = o3d.utility.Vector3dVector(colors / 255.0)
            
            # Write PLY file
            o3d.io.write_point_cloud(filename, pcd, write_ascii=False)
        except Exception as e:
            print(f"Error writing {filename}: {e}")
        finally:
            write_queue.task_done()

def gpu_inference_thread():
    """GPU inference thread - process one frame at a time"""
    frame_count = 0
    total_gpu_time = 0
    total_time = 0
    
    while True:
        frame_data = gpu_queue.get()
        
        if frame_data is None:  # Sentinel
            gpu_queue.task_done()
            break
        
        # Process single frame
        start_time = time.time()
        
        # GPU inference with known FOV
        with torch.no_grad():
            outputs = model.infer(frame_data.tensor, fov_x=fov_x_degrees)
        
        # Get RGB data from input tensor (already on GPU)
        rgb_gpu = frame_data.tensor.permute(1, 2, 0) * 255.0
        
        # Filter on GPU before CPU transfer
        valid_points, valid_colors = filter_points_gpu(
            outputs['points'], 
            outputs['mask'], 
            rgb_gpu, 
            min_distance
        )
        
        # Extract and save intrinsics
        intrinsics = outputs['intrinsics'].cpu().numpy()
        fx, fy = intrinsics[0, 0], intrinsics[1, 1]
        cx, cy = intrinsics[0, 2], intrinsics[1, 2]
        
        # Convert intrinsics to FOV (for verification)
        h, w = frame_data.tensor.shape[1:]
        fov_x_rad = 2 * np.arctan(w / (2 * fx))
        fov_y_rad = 2 * np.arctan(h / (2 * fy))
        fov_x_deg = np.rad2deg(fov_x_rad)
        fov_y_deg = np.rad2deg(fov_y_rad)
        
        # Write to CSV
        intrinsics_writer.writerow([frame_data.index, fx, fy, cx, cy, fov_x_deg, fov_y_deg])
        intrinsics_csv.flush()  # Ensure data is written immediately
        
        # Clean GPU memory
        torch.cuda.empty_cache()
        
        gpu_time = time.time() - start_time
        
        # Send filtered data to writer
        filename = os.path.join(output_folder, f'frame_{frame_data.index:04d}.ply')
        write_queue.put((filename, valid_points, valid_colors))
        
        total_frame_time = time.time() - start_time
        
        # Update statistics
        frame_count += 1
        total_gpu_time += gpu_time
        total_time += total_frame_time
        
        # Print progress every 10 frames
        if frame_count % 10 == 0:
            avg_gpu = total_gpu_time / frame_count
            avg_total = total_time / frame_count
            print(f"Processed {frame_count} frames: Avg GPU {avg_gpu:.3f}s, Avg Total {avg_total:.3f}s per frame")
            
            # Also show current GPU memory usage
            if torch.cuda.is_available():
                allocated = torch.cuda.memory_allocated() / 1024**3
                reserved = torch.cuda.memory_reserved() / 1024**3
                print(f"  GPU Memory: {allocated:.2f}GB allocated, {reserved:.2f}GB reserved")
        
        gpu_queue.task_done()
    
    # Final statistics
    if frame_count > 0:
        print(f"\nGPU thread complete: {frame_count} frames")
        print(f"Average GPU time: {total_gpu_time/frame_count:.3f}s per frame")
        print(f"Average total time: {total_time/frame_count:.3f}s per frame")
    
    # Signal end to write queue
    write_queue.put(None)

# Start threads
writer = threading.Thread(target=writer_thread, daemon=True)
gpu_thread = threading.Thread(target=gpu_inference_thread, daemon=True)

writer.start()
gpu_thread.start()

# Main loop - read video and preprocess frames
frame_idx = 0
processed_count = 0
start_total = time.time()

while cap.isOpened():
    ret, frame = cap.read()
    if not ret:
        break
    
    if frame_idx % frame_skip == 0:
        # Create frame data
        frame_data = FrameData(index=frame_idx, original=frame)
        
        # Preprocess frame
        preprocess_frame(frame_data, scale_percent)
        
        # Send to GPU queue
        gpu_queue.put(frame_data)
        processed_count += 1
    
    frame_idx += 1

# Signal end of stream
gpu_queue.put(None)

# Wait for completion
gpu_thread.join()
writer.join()

# Statistics
total_time = time.time() - start_total
print(f"\nProcessed {processed_count} frames")
print(f"Total time: {total_time:.2f}s")
print(f"Average time per frame: {total_time/processed_count:.3f}s")
print(f"Overall FPS: {processed_count/total_time:.2f}")

cap.release()
intrinsics_csv.close()
print("\nAll files written successfully!")
print("Camera intrinsics saved to: camera_intrinsics.csv")

# Download results (add to a separate cell in Colab)
# !zip -r point_clouds.zip point_clouds/
# files.download('point_clouds.zip')

Start with the supplied Colab script, especially the model.infer call using fov_x, the point-distance filtering, and the saved camera_intrinsics.csv values. Compare the returned point scale with a known distance and consult the MoGe model documentation or inference entry point; done means the source of the scale discrepancy and the correct metric-scale expectation are documented.