huggingface / huggingface/tokenizers

ModuleNotFoundError: No module named 'tokenizers.tokenizers'

Open
#1,619 6 comments 0 reactions 0 assignees View on GitHub
Dominant language
Rust
Stars
11k
Forks
1.2k
Avg merge
3d 8h
Merged PRs (30d)
26

Description

Is Transformers 4.44.0 compatible with Python 3.9.19? I receive the following error below - any thoughts are appreciated.

import transformers
from transformers import AutoTokenizer

Produces the following error:

---------------------------------------------------------------------------
ModuleNotFoundError Traceback (most recent call last)
D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in _get_module(self, module_name)
1602 try:
-> 1603 return importlib.import_module("." + module_name, self.__name__)
1604 except Exception as e:

D:\Anaconda3\lib\importlib\__init__.py in import_module(name, package)
126 level += 1
--> 127 return _bootstrap._gcd_import(name[level:], package, level)
128

D:\Anaconda3\lib\importlib\_bootstrap.py in _gcd_import(name, package, level)

D:\Anaconda3\lib\importlib\_bootstrap.py in _find_and_load(name, import_)

D:\Anaconda3\lib\importlib\_bootstrap.py in _find_and_load_unlocked(name, import_)

D:\Anaconda3\lib\importlib\_bootstrap.py in _load_unlocked(spec)

D:\Anaconda3\lib\importlib\_bootstrap_external.py in exec_module(self, module)

D:\Anaconda3\lib\importlib\_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

D:\Anaconda3\lib\site-packages\transformers\integrations\ggml.py in
23 import numpy as np
---> 24 from tokenizers import Tokenizer, decoders, normalizers, pre_tokenizers
25 from tokenizers.models import BPE

D:\Anaconda3\lib\site-packages\tokenizers\__init__.py in
77
---> 78 from .tokenizers import (
79 AddedToken,

ModuleNotFoundError: No module named 'tokenizers.tokenizers'

The above exception was the direct cause of the following exception:

RuntimeError Traceback (most recent call last)
D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in _get_module(self, module_name)
1602 try:
-> 1603 return importlib.import_module("." + module_name, self.__name__)
1604 except Exception as e:

D:\Anaconda3\lib\importlib\__init__.py in import_module(name, package)
126 level += 1
--> 127 return _bootstrap._gcd_import(name[level:], package, level)
128

D:\Anaconda3\lib\importlib\_bootstrap.py in _gcd_import(name, package, level)

D:\Anaconda3\lib\importlib\_bootstrap.py in _find_and_load(name, import_)

D:\Anaconda3\lib\importlib\_bootstrap.py in _find_and_load_unlocked(name, import_)

D:\Anaconda3\lib\importlib\_bootstrap.py in _load_unlocked(spec)

D:\Anaconda3\lib\importlib\_bootstrap_external.py in exec_module(self, module)

D:\Anaconda3\lib\importlib\_bootstrap.py in _call_with_frames_removed(f, *args, **kwds)

D:\Anaconda3\lib\site-packages\transformers\models\auto\tokenization_auto.py in
23
---> 24 from ...configuration_utils import PretrainedConfig
25 from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code

D:\Anaconda3\lib\site-packages\transformers\configuration_utils.py in
28 from .dynamic_module_utils import custom_object_save
---> 29 from .modeling_gguf_pytorch_utils import load_gguf_checkpoint
30 from .utils import (

D:\Anaconda3\lib\site-packages\transformers\modeling_gguf_pytorch_utils.py in
21
---> 22 from .integrations import (
23 GGUF_CONFIG_MAPPING,

D:\Anaconda3\lib\importlib\_bootstrap.py in _handle_fromlist(module, fromlist, import_, recursive)

D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in __getattr__(self, name)
1592 elif name in self._class_to_module.keys():
-> 1593 module = self._get_module(self._class_to_module[name])
1594 value = getattr(module, name)

D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in _get_module(self, module_name)
1604 except Exception as e:
-> 1605 raise RuntimeError(
1606 f"Failed to import {self.__name__}.{module_name} because of the following error (look up to see its"

RuntimeError: Failed to import transformers.integrations.ggml because of the following error (look up to see its traceback):
No module named 'tokenizers.tokenizers'

The above exception was the direct cause of the following exception:

RuntimeError Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_4232\77889138.py in
----> 1 from transformers import AutoTokenizer

D:\Anaconda3\lib\importlib\_bootstrap.py in _handle_fromlist(module, fromlist, import_, recursive)

D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in __getattr__(self, name)
1592 elif name in self._class_to_module.keys():
1593 module = self._get_module(self._class_to_module[name])
-> 1594 value = getattr(module, name)
1595 else:
1596 raise AttributeError(f"module {self.__name__} has no attribute {name}")

D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in __getattr__(self, name)
1591 value = self._get_module(name)
1592 elif name in self._class_to_module.keys():
-> 1593 module = self._get_module(self._class_to_module[name])
1594 value = getattr(module, name)
1595 else:

D:\Anaconda3\lib\site-packages\transformers\utils\import_utils.py in _get_module(self, module_name)
1603 return importlib.import_module("." + module_name, self.__name__)
1604 except Exception as e:
-> 1605 raise RuntimeError(
1606 f"Failed to import {self.__name__}.{module_name} because of the following error (look up to see its"
1607 f" traceback):\n{e}"

RuntimeError: Failed to import transformers.models.auto.tokenization_auto because of the following error (look up to see its traceback):
Failed to import transformers.integrations.ggml because of the following error (look up to see its traceback):
No module named 'tokenizers.tokenizers'

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.