Model initialization issue

When I tried to integrate sam2 into another model and initialize the sam2 model within the initialization function of the model class, I encountered the following error.
` def __init__(self, config, seg_token_idx=None, **kwargs):
super().__init__(config, **kwargs)

self.sam2_model = None
self.sam2_predictor = None
self.initialize_sam2()

def initialize_sam2(self):
self.sam2_model = build_sam2(model_cfg, sam2_checkpoint, device='cuda')
self.sam2_predictor = SAM2ImagePredictor(self.sam2_model)
`
`Traceback (most recent call last):
File "my_run.py", line 32, in
model = SAM2InternVLChatModel.from_pretrained(
File "/anaconda3/envs/internvl/lib/python3.9/site-packages/transformers/modeling_utils.py", line 3594, in from_pretrained
model = cls(config, *model_args, **model_kwargs)
File "chat.py", line 441, in __init__
self.initialize_sam2()
File "chat.py", line 450, in initialize_sam2
self.sam2_model = build_sam2(model_cfg, sam2_checkpoint, device='cuda')
File "model/segment-anything-2/sam2/build_sam.py", line 38, in build_sam2
model = model.to(device)
File "lib/python3.9/site-packages/torch/nn/modules/module.py", line 1145, in to
return self._apply(convert)
File "lib/python3.9/site-packages/torch/nn/modules/module.py", line 797, in _apply
module._apply(fn)
File "lib/python3.9/site-packages/torch/nn/modules/module.py", line 797, in _apply
module._apply(fn)
File "lib/python3.9/site-packages/torch/nn/modules/module.py", line 797, in _apply
module._apply(fn)
[Previous line repeated 1 more time]
File "lib/python3.9/site-packages/torch/nn/modules/module.py", line 820, in _apply
param_applied = fn(param)
File "lib/python3.9/site-packages/torch/nn/modules/module.py", line 1143, in convert
return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None, non_blocking)
NotImplementedError: Cannot copy out of meta tensor; no data!`