Train Custom Data Error

`2021-08-08 14:31:53.114 | ERROR | yolox.core.launch:launch:68 - An error has been caught in function 'launch', process 'MainProcess' (15336), thread 'MainThread' (9132):
Traceback (most recent call last):

File "D:\Anaconda\lib\site-packages\torch\utils\data\dataloader.py", line 986, in _try_get_data
data = self._data_queue.get(timeout=timeout)
│ │ │ └ 5.0
│ │ └ <function Queue.get at 0x0000025C177050D0>
│ └ <queue.Queue object at 0x0000025C1EF7BE80>
└ <torch.utils.data.dataloader._MultiProcessingDataLoaderIter object at 0x0000025C9DA38700>

File "D:\Anaconda\lib\queue.py", line 178, in get
raise Empty
└ <class '_queue.Empty'>

_queue.Empty

The above exception was the direct cause of the following exception:

Traceback (most recent call last):

File "tools/train.py", line 113, in
launch(
└ <function launch at 0x0000025C1D1931F0>

File "d:\pycharm_project\opensource\yolox\yolox\core\launch.py", line 68, in launch
main_func(*args)
│ └ (╒══════════════════╤════════════════════════════════════════════════════════════════════════════════════════════════════════...
└ <function main at 0x0000025C1F516940>

File "tools/train.py", line 101, in main
trainer.train()
│ └ <function Trainer.train at 0x0000025C1E8D2670>
└ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>

File "d:\pycharm_project\opensource\yolox\yolox\core\trainer.py", line 70, in train
self.train_in_epoch()
│ └ <function Trainer.train_in_epoch at 0x0000025C1F4D00D0>
└ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>

File "d:\pycharm_project\opensource\yolox\yolox\core\trainer.py", line 80, in train_in_epoch
self.after_epoch()
│ └ <function Trainer.after_epoch at 0x0000025C1F4DCAF0>
└ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>

File "d:\pycharm_project\opensource\yolox\yolox\core\trainer.py", line 210, in after_epoch
self.evaluate_and_save_model()
│ └ <function Trainer.evaluate_and_save_model at 0x0000025C1F4DCDC0>
└ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>

File "d:\pycharm_project\opensource\yolox\yolox\core\trainer.py", line 293, in evaluate_and_save_model
ap50_95, ap50, summary = self.exp.eval(evalmodel, self.evaluator, self.is_distributed)
│ │ │ │ │ │ │ └ False
│ │ │ │ │ │ └ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>
│ │ │ │ │ └ <yolox.evaluators.voc_evaluator.VOCEvaluator object at 0x0000025C293F65E0>
│ │ │ │ └ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>
│ │ │ └ YOLOX(
│ │ │ (backbone): YOLOPAFPN(
│ │ │ (backbone): CSPDarknet(
│ │ │ (stem): Focus(
│ │ │ (conv): BaseConv(
│ │ │ (conv): ...
│ │ └ <function Exp.eval at 0x0000025C1F516F70>
│ └ ╒══════════════════╤═════════════════════════════════════════════════════════════════════════════════════════════════════════...
└ <yolox.core.trainer.Trainer object at 0x0000025C1F51EF10>

File "d:\pycharm_project\opensource\yolox\yolox\exp\yolox_base.py", line 251, in eval
return evaluator.evaluate(model, is_distributed, half)
│ │ │ │ └ False
│ │ │ └ False
│ │ └ YOLOX(
│ │ (backbone): YOLOPAFPN(
│ │ (backbone): CSPDarknet(
│ │ (stem): Focus(
│ │ (conv): BaseConv(
│ │ (conv): ...
│ └ <function VOCEvaluator.evaluate at 0x0000025C1F4D0C10>
└ <yolox.evaluators.voc_evaluator.VOCEvaluator object at 0x0000025C293F65E0>

File "d:\pycharm_project\opensource\yolox\yolox\evaluators\voc_evaluator.py", line 82, in evaluate
for cur_iter, (imgs, _, info_imgs, ids) in enumerate(progress_bar(self.dataloader)):
│ │ │ │ │ │ │ └ <torch.utils.data.dataloader.DataLoader object at 0x0000025C293F6550>
│ │ │ │ │ │ └ <yolox.evaluators.voc_evaluator.VOCEvaluator object at 0x0000025C293F65E0>
│ │ │ │ │ └ <class 'tqdm.std.tqdm'>
│ │ │ │ └ tensor([0])
│ │ │ └ [tensor([3264]), tensor([2448])]
│ │ └ tensor([[[0., 0., 0., 0., 0.]]])
│ └ tensor([[[[ 2.0777, 2.0948, 2.0263, ..., -0.4226, 0.0398, -0.7993],
│ [ 2.0092, 2.1462, 1.9235, ..., 0.1254, ...
└ 0

File "D:\Anaconda\lib\site-packages\tqdm\std.py", line 1165, in iter
for obj in iterable:
│ └ <torch.utils.data.dataloader.DataLoader object at 0x0000025C293F6550>
└ [tensor([[[[ 2.0777, 2.0948, 2.0263, ..., -0.4226, 0.0398, -0.7993],
[ 2.0092, 2.1462, 1.9235, ..., 0.1254,...

File "D:\Anaconda\lib\site-packages\torch\utils\data\dataloader.py", line 517, in next
data = self._next_data()
│ └ <function _MultiProcessingDataLoaderIter._next_data at 0x0000025C1CCC3AF0>
└ <torch.utils.data.dataloader._MultiProcessingDataLoaderIter object at 0x0000025C9DA38700>

File "D:\Anaconda\lib\site-packages\torch\utils\data\dataloader.py", line 1182, in _next_data
idx, data = self._get_data()
│ │ └ <function _MultiProcessingDataLoaderIter._get_data at 0x0000025C1CCC3A60>
│ └ <torch.utils.data.dataloader._MultiProcessingDataLoaderIter object at 0x0000025C9DA38700>
└ 7

File "D:\Anaconda\lib\site-packages\torch\utils\data\dataloader.py", line 1138, in _get_data
success, data = self._try_get_data()
│ └ <function _MultiProcessingDataLoaderIter._try_get_data at 0x0000025C1CCC39D0>
└ <torch.utils.data.dataloader._MultiProcessingDataLoaderIter object at 0x0000025C9DA38700>

File "D:\Anaconda\lib\site-packages\torch\utils\data\dataloader.py", line 999, in _try_get_data
raise RuntimeError('DataLoader worker (pid(s) {}) exited unexpectedly'.format(pids_str)) from e
└ '10756, 12584'

RuntimeError: DataLoader worker (pid(s) 10756, 12584) exited unexpectedly`

Start with tools/train.py and the traceback path through yolox/core/trainer.py, yolox/exp/yolox_base.py, and yolox/evaluators/voc_evaluator.py. Reproduce evaluation with the custom dataset and inspect why the PyTorch DataLoader workers exit unexpectedly. Done means the cause is identified and custom-data training completes evaluation without the worker error.