BUG: VideoPredictor produced different outputs with the same model weight

I noticed that even if loading the same model weight, sam2's VideoPredictor could produce very different results.

The minimal code snippet to reproduce the bug:

In [sam2/notebooks/video_predictor_example.ipynb](https://github.com/facebookresearch/sam2/blob/main/notebooks/video_predictor_example.ipynb), add the following code block right above the section title "Initialize the inference state":

```python
ann_frame_idx = 0 # the frame index we interact with
ann_obj_id = 1 # give a unique id to each object we interact with (it can be any integers)
# Let's add a positive click at (x, y) = (210, 350) to get started
points = np.array([[210, 350]], dtype=np.float32)
# for labels, `1` means positive click and `0` means negative click
labels = np.array([1], np.int32)

def run_one_inference(predictor):
inference_state = predictor.init_state(video_path=video_dir)
predictor.reset_state(inference_state)
_, _, out_mask_logits = predictor.add_new_points_or_box(
inference_state=inference_state,
frame_idx=ann_frame_idx,
obj_id=ann_obj_id,
points=points,
labels=labels,
)
return out_mask_logits[0]

# silently run one inference
predictor1 = build_sam2_video_predictor(model_cfg, ckpt_path=None, device=device)
run_one_inference(predictor1) # comment out this line will make mask1 and mask2 identical
predictor1.load_state_dict(torch.load(sam2_checkpoint)['model'],strict=True)
mask1 = run_one_inference(predictor1)

predictor2 = build_sam2_video_predictor(model_cfg, ckpt_path=sam2_checkpoint, device=device)
mask2 = run_one_inference(predictor2)

print(torch.allclose(mask1,mask2))
```

In theory, mask1 and mask2 should be the same, as predictor1 and predictor2 loaded the same weight, but mask1 and mask2 are very different.

I have used the following function to check whether predictor1 and predictor2 are the same, and they seem to be the same.
```python
import inspect
def diff_non_state_attributes(m1, m2, ignore_private=True):
"""
Compare all attributes of m1 and m2 that are not in state_dict.
This will help find persistent state or caches that differ.
"""
sd_keys = set(m1.state_dict().keys())

def collect_attrs(model):
attrs = {}
for name, val in inspect.getmembers(model):
# Ignore methods, modules, and state_dict entries
if ignore_private and name.startswith("_"):
continue
if callable(val) or isinstance(val, torch.nn.Module):
continue
if name in sd_keys:
continue
# We skip Tensors already in state_dict
attrs[name] = val
return attrs

attrs1 = collect_attrs(m1)
attrs2 = collect_attrs(m2)

diffs = []
for k in sorted(set(attrs1.keys()) | set(attrs2.keys())):
v1 = attrs1.get(k, "")
v2 = attrs2.get(k, "")
if isinstance(v1, torch.Tensor) and isinstance(v2, torch.Tensor):
if not torch.equal(v1, v2):
diffs.append((k, f"TENSOR diff, norm={torch.norm(v1 - v2).item():.3e}"))
else:
if v1 != v2:
diffs.append((k, f"{v1} != {v2}"))
return diffs

def diff_state(m1, m2, rtol=1e-5, atol=1e-7):
# Map to know which keys are params vs buffers
p1 = {k: v for k, v in m1.named_parameters()}
b1 = {k: v for k, v in m1.named_buffers()}

sd1 = m1.state_dict() # includes params + buffers
sd2 = m2.state_dict()

diffs = []
for k in sd1.keys():
if k not in sd2:
diffs.append((k, 'missing_in_m2', None))
continue
t1, t2 = sd1[k], sd2[k]
if t1.shape != t2.shape:
diffs.append((k, 'shape_mismatch', (t1.shape, t2.shape)))
continue
# allclose check
if not torch.allclose(t1, t2, rtol=rtol, atol=atol):
kind = 'param' if k in p1 else ('buffer' if k in b1 else 'unknown')
diffs.append((k, kind, torch.norm((t1 - t2).float()).item()))
return diffs

def pretty_print_diffs(diffs):
if not diffs:
print("No differences (within tolerance).")
return
print("Differences:")
for k, kind, info in diffs:
if info is None:
print(f" {k:50s} [{kind}]")
elif isinstance(info, tuple):
print(f" {k:50s} [{kind}] shape {info[0]} vs {info[1]}")
else:
print(f" {k:50s} [{kind}] ||Δ||₂ = {info:.3e}")

diffs = diff_state(predictor1,predictor2)
pretty_print_diffs(diffs)
diffs = diff_non_state_attributes(predictor1,predictor2,ignore_private=True)
if not diffs:
print("No non-state attributes differ.")
else:
print("Differences in non-state attributes:")
for k, info in diffs:
print(f" {k}: {info}")
```
I have no idea what happens, and have been stuck with this bug for one week.