in place

Hello, I want to replace the model opt1.3b with baichuan 7b and change the model path. The keyword "name or path":"/T106/LLM model/Baichuan-7B" was added to the config.josn file of Baichuan model。now I can not guide into the Tokenizern error。The following is the source code and error content。

util.py
def load_hf_tokenizer(model_name_or_path, fast_tokenizer=True):
print("hello world")
if os.path.exists(model_name_or_path):
# Locally tokenizer loading has some issue, so we need to force download
model_json = os.path.join(model_name_or_path, "config.json")
if os.path.exists(model_json):
print(model_json)
model_json_file = json.load(open(model_json))
print(model_json_file)
print(model_name_or_path)
model_name = model_json_file.get("_name_or_path",
model_name_or_path)
print(f"AAAAAAAAAAAAAAAAA{model_name}")
tokenizer = get_tokenizer(model_name,
fast_tokenizer=fast_tokenizer)
print(f"FFFFFFFFFFFFFFFF{model_name}")
else:
tokenizer = get_tokenizer(model_name_or_path,
fast_tokenizer=fast_tokenizer)

return tokenizer

mian.py
tokenizer = load_hf_tokenizer(args.model_name_or_path, fast_tokenizer=True)

config.json #baichuan-7b model
{
"_name_or_path":"/T106/LLM_model/Baichuan-7B",
"architectures": [
"BaiChuanForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_baichuan.BaiChuanConfig",
"AutoModelForCausalLM": "modeling_baichuan.BaiChuanForCausalLM"
},
"bos_token_id": 1,
"eos_token_id": 2,
"hidden_act": "silu",
"hidden_size": 4096,
"initializer_range": 0.02,
"intermediate_size": 11008,
"max_position_embeddings": 4096,
"model_type": "baichuan",
"num_attention_heads": 32,
"num_hidden_layers": 32,
"pad_token_id": 0,
"rms_norm_eps": 1e-06,
"tie_word_embeddings": false,
"torch_dtype": "float32",
"transformers_version": "4.29.1",
"use_cache": true,
"vocab_size": 64000
}
tokenizer_config.json
{
"auto_map": {
"AutoTokenizer": ["tokenization_baichuan.BaiChuanTokenizer", null]
},
"add_bos_token": false,
"add_eos_token": false,
"bos_token": {
"__type": "AddedToken",
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"name_or_path": "/T106/LLM_model/Baichuan-7B",
"clean_up_tokenization_spaces": false,
"eos_token": {
"__type": "AddedToken",
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"model_max_length": 1000000000000000019884624838656,
"sp_model_kwargs": {},
"tokenizer_class": "BaiChuanTokenizer",
"unk_token": {
"__type": "AddedToken",
"content": "",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}

[2023-10-16 02:06:58,323] [INFO] [real_accelerator.py:158:get_accelerator] Setting ds_accelerator to cuda (auto detect)
/opt/conda/lib/python3.8/site-packages/transformers/deepspeed.py:23: FutureWarning: transformers.deepspeed module is deprecated and will be removed in a future version. Please import deepspeed modules directly from transformers.integrations
warnings.warn(
[2023-10-16 02:07:00,534] [INFO] [comm.py:637:init_distributed] cdb=None
[2023-10-16 02:07:00,566] [INFO] [comm.py:668:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl
hello world
/T106/LLM_model/Baichuan-7B/config.json
{'_name_or_path': '/T106/LLM_model/Baichuan-7B', 'architectures': ['BaiChuanForCausalLM'], 'auto_map': {'AutoConfig': 'configuration_baichuan.BaiChuanConfig', 'AutoModelForCausalLM': 'modeling_baichuan.BaiChuanForCausalLM'}, 'bos_token_id': 1, 'eos_token_id': 2, 'hidden_act': 'silu', 'hidden_size': 4096, 'initializer_range': 0.02, 'intermediate_size': 11008, 'max_position_embeddings': 4096, 'model_type': 'baichuan', 'num_attention_heads': 32, 'num_hidden_layers': 32, 'pad_token_id': 0, 'rms_norm_eps': 1e-06, 'tie_word_embeddings': False, 'torch_dtype': 'float32', 'transformers_version': '4.29.1', 'use_cache': True, 'vocab_size': 64000}
/T106/LLM_model/Baichuan-7B
AAAAAAAAAAAAAAAAA/T106/LLM_model/Baichuan-7B
Traceback (most recent call last):
File "main.py", line 397, in
main()
File "main.py", line 245, in main
tokenizer = load_hf_tokenizer(args.model_name_or_path, fast_tokenizer=True)
File "/T106/model_main/DeepSpeedExamples-master/DeepSpeedExamples-master/applications/DeepSpeed-Chat/training/utils/utils.py", line 79, in load_hf_tokenizer
tokenizer = get_tokenizer(model_name,
File "/T106/model_main/DeepSpeedExamples-master/DeepSpeedExamples-master/applications/DeepSpeed-Chat/training/utils/utils.py", line 58, in get_tokenizer
tokenizer = AutoTokenizer.from_pretrained(
File "/opt/conda/lib/python3.8/site-packages/transformers/models/auto/tokenization_auto.py", line 733, in from_pretrained
raise ValueError(
ValueError: Tokenizer class BaiChuanTokenizer does not exist or is not currently imported.
[2023-10-16 02:07:04,159] [INFO] [launch.py:315:sigkill_handler] Killing subprocess 35631
[2023-10-16 02:07:04,159] [ERROR] [launch.py:321:sigkill_handler] ['/opt/conda/bin/python3.8', '-u', 'main.py', '--local_rank=0', '--model_name_or_path', '/T106/LLM_model/Baichuan-7B', '--gradient_accumulation_steps', '8', '--lora_dim', '128', '--zero_stage', '0', '--enable_tensorboard', '--tensorboard_path', './output', '--deepspeed', '--output_dir', './output'] exits with return code = 1

### Tasks

Start in training/utils/utils.py at load_hf_tokenizer and get_tokenizer, then follow the call from main.py. Reproduce the failure with the provided local Baichuan-7B config.json and tokenizer_config.json, and inspect the AutoTokenizer loading path. Done means the local model's tokenizer initializes successfully without the reported BaiChuanTokenizer error.