ScanNet++V2 Benchmarking Discrepancy

Hey folks, congrats on the amazing release!

I wanted to follow up regarding the benchmarking trends you showed for MapAnything on ScanNet++ in your paper. The bad performance of MapAnything on DTU checks out with my internal testing due to the released MapAnything not being trained on object centric data and it's metric training priors not aligning well with the toy scale objects presented in the DTU dataset.

However, the ScanNet++ and other indoor dataset trends don't seem to align with my intuition. I wonder if this difference stems from how your ScanNet++ dataset is processed? Do you folks use the iphone RGB images and poses (I believe these are lower quality in terms of precision)?

**I've added Depth Anything 3 to the MapAnything benchmarking framework and I see very impressive results on ETH3D and subpar performance on ScanNet++V2 and TartanAirV2-WB:**

**Average:**

**ETH3D:**

**ScanNet++V2:**

**TartanAirV2:**

Also I visualized the outputs (on datasets and in-the-wild image sets) and think that there is no bug with the benchmarking/inference. The code for your reference (I'm testing with 504 longest side and dinov2 image normalization):
```
# Copyright (c) Meta Platforms, Inc. and affiliates.

# This source code is licensed under the Apache License, Version 2.0
# found in the LICENSE file in the root directory of this source tree.

"""
Inference wrapper for Depth Anything 3
"""

import numpy as np
import torch

from depth_anything_3.api import DepthAnything3

from mapanything.models.external.vggt.utils.geometry import closed_form_inverse_se3
from mapanything.models.external.vggt.utils.rotation import mat_to_quat
from mapanything.utils.geometry import (
convert_ray_dirs_depth_along_ray_pose_trans_quats_to_pointmap,
convert_z_depth_to_depth_along_ray,
depthmap_to_camera_frame,
get_rays_in_camera_frame,
)

class DA3Wrapper(torch.nn.Module):
def __init__(
self,
name,
torch_hub_force_reload,
hf_model_name,
):
super().__init__()
self.name = name
self.torch_hub_force_reload = torch_hub_force_reload
self.hf_model_name = hf_model_name

# Load pre-trained weights
if not torch_hub_force_reload:
# Initialize the DA3 model from huggingface hub cache
print("Loading DA3 from huggingface cache ...")
self.model = DepthAnything3.from_pretrained(
self.hf_model_name,
)
else:
# Initialize the DA3 model
self.model = DepthAnything3.from_pretrained(self.hf_model_name, force_download=True)

# Get the dtype for DA3 inference
# bfloat16 is supported on Ampere GPUs (Compute Capability 8.0+)
self.dtype = (
torch.bfloat16
if torch.cuda.get_device_capability()[0] >= 8
else torch.float16
)

def forward(self, views):
"""
Forward pass wrapper for DA3

Assumption:
- All the input views have the same image shape.

Args:
views (List[dict]): List of dictionaries containing the input views' images and instance information.
Each dictionary should contain the following keys:
"img" (tensor): Image tensor of shape (B, C, H, W).
"data_norm_type" (list): ["dinov2"]

Returns:
List[dict]: A list containing the final outputs for all N views.
"""
# Get input shape of the images, number of views, and batch size per view
batch_size_per_view, _, height, width = views[0]["img"].shape
num_views = len(views)

# Check the data norm type
data_norm_type = views[0]["data_norm_type"][0]
assert data_norm_type == "dinov2", (
"DA3 expects DINOv2 normalization for the input images"
)

# Concatenate the images to create a single (B, V, C, H, W) tensor
img_list = [view["img"] for view in views]
images = torch.stack(img_list, dim=1)

# Run the DA3 model
with torch.autocast("cuda", dtype=self.dtype):
results = self.model(images, export_feat_layers=[])

# Need high precision for transformations
with torch.autocast("cuda", enabled=False):
res = []
for view_idx in range(num_views):
# Get the extrinsics, intrinsics, depth map for the current view
curr_view_extrinsic = results["extrinsics"][:, view_idx, ...]
curr_view_extrinsic = closed_form_inverse_se3(
curr_view_extrinsic
) # Convert to cam2world
curr_view_intrinsic = results["intrinsics"][:, view_idx, ...]
curr_view_depth_z = results["depth"][:, view_idx, ...]
curr_view_depth_z = curr_view_depth_z.squeeze(-1)
curr_view_confidence = results["depth_conf"][:, view_idx, ...]

# Get the camera frame pointmaps
curr_view_pts3d_cam, _ = depthmap_to_camera_frame(
curr_view_depth_z, curr_view_intrinsic
)

# Convert the extrinsics to quaternions and translations
curr_view_cam_translations = curr_view_extrinsic[..., :3, 3]
curr_view_cam_quats = mat_to_quat(curr_view_extrinsic[..., :3, :3])

# Convert the z depth to depth along ray
curr_view_depth_along_ray = convert_z_depth_to_depth_along_ray(
curr_view_depth_z, curr_view_intrinsic
)
curr_view_depth_along_ray = curr_view_depth_along_ray.unsqueeze(-1)

# Get the ray directions on the unit sphere in the camera frame
_, curr_view_ray_dirs = get_rays_in_camera_frame(
curr_view_intrinsic, height, width, normalize_to_unit_sphere=True
)

# Get the pointmaps
curr_view_pts3d = (
convert_ray_dirs_depth_along_ray_pose_trans_quats_to_pointmap(
curr_view_ray_dirs,
curr_view_depth_along_ray,
curr_view_cam_translations,
curr_view_cam_quats,
)
)

# Append the outputs to the result list
res.append(
{
"pts3d": curr_view_pts3d,
"pts3d_cam": curr_view_pts3d_cam,
"ray_directions": curr_view_ray_dirs,
"depth_along_ray": curr_view_depth_along_ray,
"cam_trans": curr_view_cam_translations,
"cam_quats": curr_view_cam_quats,
"conf": curr_view_confidence,
}
)

return res
```