NER ml backend doesn't display predictions ([label_studio_ml.model::get_result_from_last_job::131] 1668451470 job returns exception: )

My code :

```
class SpacyModel(LabelStudioMLBase):

TRAIN_EVENTS = ()
def __init__(self, **kwargs):
super(SpacyModel, self).__init__(**kwargs)

self.model = self.load()
self.model_version = self.train_output['checkpoint'] if 'checkpoint' in self.train_output else 'fallback'

logger.info("MODEL CHECKPOINT: %s", self.model_version)

def misc_labels(self):
map = {}
from_names = [name for names in LABEL_CONFIG.values()
for name in names]
for from_name, schema in self.parsed_label_config.items():
if from_name in from_names:
continue

for label in schema['labels']:
map[label] = {
'from_name': from_name,
'to_name': schema['to_name'][0],
}
return map

def ner_labels(self):
return self.misc_labels() | label_dict_from_config(
self.parsed_label_config, LABEL_CONFIG['ner'])

def spancat_labels(self):
return self.misc_labels() | label_dict_from_config(
self.parsed_label_config, LABEL_CONFIG['spancat'])

def textcat_labels(self):
return self.misc_labels() | label_dict_from_config(
self.parsed_label_config, LABEL_CONFIG['textcat'])

def load(self):
model_dir = os.path.dirname(os.path.realpath(__file__))
fallback_dir = os.path.join(model_dir, "model-best")

if PREDICTION_GPU_ID > -1:
spacy.prefer_gpu(gpu_id=PREDICTION_GPU_ID)

if 'model_path' in self.train_output and os.path.isdir(self.train_output['model_path']):
return spacy.load(self.train_output['model_path'])
elif os.path.isdir(fallback_dir):
return spacy.load(fallback_dir)

return None

def predict(self, tasks, **kwargs):

print('oussama1')
if not self.model:
logger.error("model has not been trained yet")
return {}

ner_labels = self.ner_labels()
spancat_labels = self.spancat_labels()
textcat_labels = self.textcat_labels()
predictions = []

docs = self.model.pipe([t['data']['text']
for t in tasks], batch_size=PREDICTION_BATCH_SIZE)
for doc in docs:
results = []

for e in doc.ents:
config = ner_labels[e.label_]
results.append({
'from_name': config['from_name'],
'to_name': config['to_name'],
'type': 'labels',
'value': {
'start': e.start_char,
'end': e.end_char,
'text': e.text,
'labels': [e.label_]
}
})

for SPANCAT_KEY in doc.spans:
for span in doc.spans[SPANCAT_KEY]:
config = spancat_labels[span.label_]
results.append({
'from_name': config['from_name'],
'to_name': config['to_name'],
'type': 'labels',
'value': {
'start': span.start_char,
'end': span.end_char,
'text': span.text,
'labels': [span.label_]
}
})

choices = [choice for choice, score in doc.cats.items()
if score >= TEXTCAT_SCORE_THRESHOLD]
if len(choices) > 0:
config = textcat_labels[choices[0]]
results.append({
'from_name': config['from_name'],
'to_name': config['to_name'],
'type': 'choices',
'value': {
'choices': choices
}
})

predictions.append({
'model_version': self.model_version,
'result': results
})

return predictions

def fit(self, annotations, workdir=None, **kwargs):

model_dir = os.path.dirname(os.path.realpath(__file__))
checkpoint_name = datetime.now().strftime("%Y%m%d%H%M%S")
checkpoint_dir = os.path.join(
model_dir, 'checkpoints', checkpoint_name)
config_path = os.path.join(model_dir, 'config.cfg')

train_data_path = os.path.join(checkpoint_dir, 'train.spacy')
dev_data_path = os.path.join(checkpoint_dir, 'dev.spacy')
model_path = os.path.join(checkpoint_dir, 'model-best')
latest_path = os.path.join(model_dir, "latest-model")
latest_path_tmp = os.path.join(model_dir, "latest-model-tmp")

Path(checkpoint_dir).mkdir(parents=True, exist_ok=True)

annotations = list(filter(item_not_cancelled, list(annotations)))

train_data, dev_data = split_annotations(annotations, EVAL_SPLIT)

annotations_to_docbin(
train_data,
ner_labels=self.ner_labels(),
spancat_labels=self.spancat_labels(),
textcat_labels=self.textcat_labels()
).to_disk(train_data_path)

annotations_to_docbin(
dev_data,
ner_labels=self.ner_labels(),
spancat_labels=self.spancat_labels(),
textcat_labels=self.textcat_labels()
).to_disk(dev_data_path)

if TRAIN_GPU_ID > -1:
try:
import gc
import torch
self.model = None
gc.collect()
torch.cuda.empty_cache()
except:
pass

train(config_path, checkpoint_dir, use_gpu=TRAIN_GPU_ID, overrides={
'paths.train': train_data_path, 'paths.dev': dev_data_path})

os.symlink(model_path, latest_path_tmp)
os.replace(latest_path_tmp, latest_path)
print('oussama2')

return {'model_path': model_path, 'checkpoint': checkpoint_name}
```

```
ℹ Saving to output directory:
/home/oussama/my-ml-backend0/checkpoints/20221114204115
ℹ Using CPU

=========================== Initializing pipeline ===========================
✔ Initialized pipeline

============================= Training pipeline =============================
ℹ Pipeline: ['tok2vec', 'ner']
ℹ Initial learn rate: 0.001
E # LOSS TOK2VEC LOSS NER ENTS_F ENTS_P ENTS_R SCORE
--- ------ ------------ -------- ------ ------ ------ ------
0 0 0.00 11.94 0.00 0.00 0.00 0.00
200 200 0.68 206.59 0.00 0.00 0.00 0.00
400 400 0.00 0.00 0.00 0.00 0.00 0.00
✔ Saved pipeline to output directory
/home/oussama/my-ml-backend0/checkpoints/20221114204115/model-last
oussama2
[2022-11-14 20:41:24,480] [ERROR] [label_studio_ml.exceptions::exception_f::53] Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/exceptions.py", line 39, in exception_f
return f(*args, **kwargs)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/api.py", line 93, in _train
job = _manager.train(annotations, project, label_config, **params)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 714, in train
cls.get_or_create(project, label_config, force_reload=True, train_output=train_output)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 490, in get_or_create
if not cls.has_active_model(project) or \
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 452, in has_active_model
return cls._key(project) in cls._current_model
TypeError: argument of type 'ModelWrapper' is not iterable

Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/exceptions.py", line 39, in exception_f
return f(*args, **kwargs)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/api.py", line 93, in _train
job = _manager.train(annotations, project, label_config, **params)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 714, in train
cls.get_or_create(project, label_config, force_reload=True, train_output=train_output)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 490, in get_or_create
if not cls.has_active_model(project) or \
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 452, in has_active_model
return cls._key(project) in cls._current_model
TypeError: argument of type 'ModelWrapper' is not iterable

[2022-11-14 20:41:24,480] [INFO] [werkzeug::_log::225] 127.0.0.1 - - [14/Nov/2022 20:41:24] "POST /train HTTP/1.1" 500 -
[2022-11-14 20:41:24,504] [INFO] [werkzeug::_log::225] 127.0.0.1 - - [14/Nov/2022 20:41:24] "GET /health HTTP/1.1" 200 -
[2022-11-14 20:41:24,509] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451470 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,510] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451450 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,510] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451120 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,511] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451116 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,511] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451088 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,511] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451068 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,511] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450840 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,511] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450835 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,511] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450218 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,512] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450217 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,512] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450211 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,512] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450210 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:24,512] [INFO] [model::__init__::55] MODEL CHECKPOINT: fallback
[2022-11-14 20:41:24,513] [INFO] [werkzeug::_log::225] 127.0.0.1 - - [14/Nov/2022 20:41:24] "POST /setup HTTP/1.1" 200 -
[2022-11-14 20:41:31,135] [INFO] [werkzeug::_log::225] 127.0.0.1 - - [14/Nov/2022 20:41:31] "GET /health HTTP/1.1" 200 -
[2022-11-14 20:41:31,140] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451470 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451450 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451120 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451116 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451088 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668451068 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450840 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450835 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,141] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450218 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,142] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450217 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,142] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450211 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,142] [ERROR] [label_studio_ml.model::get_result_from_last_job::131] 1668450210 job returns exception:
Traceback (most recent call last):
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 129, in get_result_from_last_job
result = self.get_result_from_job_id(job_id)
File "/home/oussama/Downloads/label-studio-ml-backend/label_studio_ml/model.py", line 111, in get_result_from_job_id
assert isinstance(result, dict)
AssertionError
[2022-11-14 20:41:31,142] [INFO] [model::__init__::55] MODEL CHECKPOINT: fallback
[2022-11-14 20:41:31,143] [INFO] [werkzeug::_log::225] 127.0.0.1 - - [14/Nov/2022 20:41:31] "POST /setup HTTP/1.1" 200 -
```

these are the data I'm using, only to test if everything is working

[
{
"data": {
"text": "More rapid margin progress."
}}
,
{
"data": {
"text": "Demonstration of gross margin and EBITA margin progression. Better than GDP+ growth."
}}
]

It seems that once the model is trained, the backend isn't able to capture the predicted data, Please if anyone can help I will highly appreciate it

Start in label_studio_ml/model.py at has_active_model, get_or_create, and get_result_from_job_id, then trace the /train flow in label_studio_ml/api.py. Reproduce the reported training failure and inspect the related exception handling in label_studio_ml/exceptions.py. Done means training completes without the ModelWrapper TypeError and predictions are returned instead of an assertion failure.

Written by the indexing model from the issue text.