InternLM / InternLM/xtuner

internvl_v2_internlm2_26b_qlora_finetune 报错:libtorch_cuda_cpp.so: cannot open shared object file: No such file or directory

Open
#866 0 comments 0 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
5.2k
Forks
448
Avg merge
3d 15h
Merged PRs (30d)
26

Description

命令:xtuner train internvl_v2_internlm2_26b_qlora_finetune.py -deepspeed deepspeed_zero2
报错:libtorch_cuda_cpp.so: cannot open shared object file: No such file or directory
File "/root/miniconda3/lib/python3.10/importlib/__init__.py", line 126, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "/root/miniconda3/lib/python3.10/site-packages/apex/normalization/fused_layer_norm.py", line 364, in __init__
fused_layer_norm_cuda = importlib.import_module("fused_layer_norm_cuda")
File "/root/.cache/huggingface/modules/transformers_modules/InternVL2-26B/modeling_intern_vit.py", line 208, in __init__
self.q_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
File "/root/.cache/huggingface/modules/transformers_modules/InternVL2-26B/modeling_intern_vit.py", line 278, in __init__
self.attn = InternAttention(config)
File "/root/.cache/huggingface/modules/transformers_modules/InternVL2-26B/modeling_intern_vit.py", line 319, in
InternVisionEncoderLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)])
File "/root/.cache/huggingface/modules/transformers_modules/InternVL2-26B/modeling_intern_vit.py", line 318, in __init__
self.layers = nn.ModuleList([
File "/root/.cache/huggingface/modules/transformers_modules/InternVL2-26B/modeling_intern_vit.py", line 379, in __init__
self.encoder = InternVisionEncoder(config)
File "/root/.cache/huggingface/modules/transformers_modules/InternVL2-26B/modeling_internvl_chat.py", line 58, in __init__
self.vision_model = InternVisionModel(config.vision_config)
File "/root/miniconda3/lib/python3.10/site-packages/transformers/modeling_utils.py", line 3404, in from_pretrained
model = cls(config, *model_args, **model_kwargs)
File "/root/miniconda3/lib/python3.10/site-packages/transformers/models/auto/auto_factory.py", line 558, in from_pretrained
return model_class.from_pretrained(
File "/root/miniconda3/lib/python3.10/site-packages/xtuner/model/internvl.py", line 73, in __init__
self.model = AutoModel.from_pretrained(
File "/root/miniconda3/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 121, in build_from_cfg
obj = obj_cls(**args) # type: ignore
File "/root/miniconda3/lib/python3.10/site-packages/mmengine/registry/build_functions.py", line 232, in build_model_from_cfg
return build_from_cfg(cfg, registry, default_args)
File "/root/miniconda3/lib/python3.10/site-packages/mmengine/registry/registry.py", line 570, in build
return self.build_func(cfg, *args, **kwargs, registry=self)
File "/root/miniconda3/lib/python3.10/site-packages/mmengine/_strategy/base.py", line 306, in build_model
model = MODELS.build(model)
File "/root/miniconda3/lib/python3.10/site-packages/mmengine/_strategy/deepspeed.py", line 381, in prepare
model = self.build_model(model)
File "/root/miniconda3/lib/python3.10/site-packages/mmengine/runner/_flexible_runner.py", line 1182, in train
self.strategy.prepare(
File "/xtuner/tools/train.py", line 356, in main
runner.train()
File "/xtuner/tools/train.py", line 360, in
main()
File "/root/miniconda3/lib/python3.10/runpy.py", line 86, in _run_code
exec(code, run_globals)
File "/root/miniconda3/lib/python3.10/runpy.py", line 196, in _run_module_as_main (Current frame)
return _run_code(code, main_globals, None,

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.