IsADirectoryError: [Errno 21] Is a directory: '/data/bdlml/mkabra/nltk_data/corpora/stopwords'
Nobody has claimed this yet.
- Dominant language
- Python
- Stars
- 4.2k
- Forks
- 385
- PR merge metrics
- No merged PRs in 30d
Description
I am getting this error while using keybert, and I have no idea on how to resolve it.
IsADirectoryError: [Errno 21] Is a directory: '/data/bdlml/mkabra/nltk_data/corpora/stopwords'
I deleted the directory, and then it says OSError: No such file or directory: '/data/bdlml/mkabra/nltk_data/corpora/stopwords'
Can you please help with it?
```
Extracting KeyPhrases: 0%| | 0/1 [00:00
45 df.iloc[i*batch_size:min(len(text_list), (i+1)*batch_size), -1] = [output]
46 else:
---> 47 df.iloc[i*batch_size:min(len(text_list), (i+1)*batch_size), -1] = KPE.extract_phrases_all(docs)
48
49 df['phrases_all_cmpln_summary'] = df['phrases_all_cmpln_summary'].astype(str)
/tmp/ipykernel_49851/4174669147.py in extract_phrases_all(self, x, top)
42 def extract_phrases_all(self, x, top=20):
43 x = self.preprocess(x)
---> 44 keyphrases = self.kw_model.extract_keywords(x, vectorizer=self.kcv, top_n=top, stop_words=None, use_mmr=True, diversity=0.7)
45 return keyphrases
46
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/keybert/_model.py in extract_keywords(self, docs, candidates, keyphrase_ngram_range, stop_words, top_n, min_df, use_maxsum, use_mmr, diversity, nr_candidates, vectorizer, highlight, seed_keywords, doc_embeddings, word_embeddings)
144 # Extract potential words using a vectorizer / tokenizer
145 if vectorizer:
--> 146 count = vectorizer.fit(docs)
147 else:
148 try:
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/keyphrase_vectorizers/keyphrase_tfidf_vectorizer.py in fit(self, raw_documents)
204
205 self._check_params()
--> 206 X = super().fit_transform(raw_documents)
207 self._tfidf.fit(X)
208 return self
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/keyphrase_vectorizers/keyphrase_count_vectorizer.py in fit_transform(self, raw_documents)
220
221 # fit
--> 222 KeyphraseCountVectorizer.fit(self=self, raw_documents=raw_documents)
223
224 # transform
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/keyphrase_vectorizers/keyphrase_count_vectorizer.py in fit(self, raw_documents)
167 lowercase=self.lowercase, workers=self.workers,
168 spacy_exclude=self.spacy_exclude,
--> 169 custom_pos_tagger=self.custom_pos_tagger)
170
171 # remove keyphrases that have more than 8 words, as they are probably no real keyphrases
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/keyphrase_vectorizers/keyphrase_vectorizer_mixin.py in _get_pos_keyphrases(self, document_list, stop_words, spacy_pipeline, pos_pattern, spacy_exclude, custom_pos_tagger, lowercase, workers)
286 if isinstance(stop_words, str):
287 try:
--> 288 stop_words_list = set(nltk.corpus.stopwords.words(stop_words))
289 except LookupError:
290 logger = logging.getLogger('KeyphraseVectorizer')
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/nltk/corpus/reader/wordlist.py in words(self, fileids, ignore_lines_startswith)
19 return [
20 line
---> 21 for line in line_tokenize(self.raw(fileids))
22 if not line.startswith(ignore_lines_startswith)
23 ]
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/nltk/corpus/reader/api.py in raw(self, fileids)
216 contents = []
217 for f in fileids:
--> 218 with self.open(f) as fp:
219 contents.append(fp.read())
220 return concat(contents)
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/nltk/corpus/reader/api.py in open(self, file)
229 """
230 encoding = self.encoding(file)
--> 231 stream = self._root.join(file).open(encoding)
232 return stream
233
~/mls_python_venv/miniconda/envs/mk_ve_3/lib/python3.7/site-packages/nltk/data.py in open(self, encoding)
322
323 def open(self, encoding=None):
--> 324 stream = open(self._path, "rb")
325 if encoding is not None:
326 stream = SeekableUnicodeStreamReader(stream, encoding)
IsADirectoryError: [Errno 21] Is a directory: '/data/bdlml/mkabra/nltk_data/corpora/stopwords'
```
Code
```
class KeyPhraseExtractor():
def __init__(self, kw_model):
self.kw_model = kw_model
self.kcv = KeyphraseTfidfVectorizer(stop_words='', pos_pattern = '*+<.*>{0,2}+<.*>{0,2}+|++<.*>{0,2}*+|+<.*>{0,2}*+|<.*>{0,2}*+<.*>{0,2}+|*+<.*>{0,2}+|<.*>{0,2}+*+<.*>{0,2}*' )
def preprocess(self, x):
if type(x) == list:
for i in range(len(x)):
x[i] = str(x[i]).replace(".",". ").replace("//",". ")
else:
x = str(x).replace(".",". ").replace("//",". ")
return x
return keyphrases
def extract_phrases_all(self, x, top=20):
x = self.preprocess(x)
keyphrases = self.kw_model.extract_keywords(x, vectorizer=self.kcv, top_n=top, stop_words=None, use_mmr=True, diversity=0.7)
return keyphrases
# Function describing the list of accepted arguments
kw_model = KeyBERT(model='ProsusAI/finbert')
df=pd.read_csv(input_path+input_file+'.csv')
df=df.loc[:10]
print(len(df))
column = 'cmpln_summary'
new_column = 'cmpln_summary'
#nlp = spacy.load("en_core_web_sm")
KPE = KeyPhraseExtractor(kw_model)
df['phrases_all_cmpln_summary'] = None
batch_size = 256
text_list = df[new_column].astype(str).tolist()
for i in tqdm(range(len(text_list) // batch_size + 1), desc='Extracting KeyPhrases'):
docs = text_list[i*batch_size:(i+1)*batch_size]
if len(docs) == 1:
output = KPE.extract_phrases_all(docs[0])
if output == []:
df.iloc[i*batch_size:min(len(text_list), (i+1)*batch_size), -1] = [['None Found']]
else:
df.iloc[i*batch_size:min(len(text_list), (i+1)*batch_size), -1] = [output]
else:
df.iloc[i*batch_size:min(len(text_list), (i+1)*batch_size), -1] = KPE.extract_phrases_all(docs)
```
Contributor guide
No contributing guide indexed for this repository
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Start with the traceback through KeyphraseTfidfVectorizer.fit and the stop_words='' setting in the provided KeyPhraseExtractor. Reproduce the error with the shown KeyBERT and Python setup; done should be a confirmed cause and a concrete repository change or documentation clarification.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- python
- Domain
- machine-learning
- Issue type
- Bug
- Difficulty
- 3/5
- Estimated time
- 1-2 days
- Activity status
- Stale
- Clarity
- Needs clarification
- Newbie friendliness
- 25/100