`everyvoice check-data` causes RuntimeError

Nobody has claimed this yet.

See note from @marctessier:

I am notcing , depending on the dataset size, / number of CPU being used , process would randomly crash . with too many open files.

```
RuntimeError: Too many open files. Communication with the workers is no longer
possible. Please increase the limit using `ulimit -n` in the shell or change the
sharing strategy by calling
`torch.multiprocessing.set_sharing_strategy('file_system')` at the beginning of
your code
```

( see below for full log)

```
╭───────────────────── Traceback (most recent call last) ──────────────────────╮
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_pr606/everyv │
│ oice/model/feature_prediction/FastSpeech2_lightning/fs2/cli/check_data.py:18 │
│ 8 in check_data_command │
│ │
│ 185 │ │ # We can't just use model.global_step because it gets reset by │
│ 186 │ │ global_step = get_global_step(model_path) │
│ 187 │ │ │
│ ❱ 188 │ │ synthesize_helper( │
│ 189 │ │ │ model=model, │
│ 190 │ │ │ texts=None, │
│ 191 │ │ │ style_reference=style_reference, │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/TxT2SPEECH/EveryVoice_pr606/everyv │
│ oice/model/feature_prediction/FastSpeech2_lightning/fs2/cli/synthesize.py:34 │
│ 9 in synthesize_helper │
│ │
│ 346 │ return ( │
│ 347 │ │ model.config, │
│ 348 │ │ device, │
│ ❱ 349 │ │ trainer.predict( │
│ 350 │ │ │ model, │
│ 351 │ │ │ FastSpeech2SynthesisDataModule( │
│ 352 │ │ │ │ model.config, │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py:887 in │
│ predict │
│ │
│ 884 │ │ self.state.fn = TrainerFn.PREDICTING │
│ 885 │ │ self.state.status = TrainerStatus.RUNNING │
│ 886 │ │ self.predicting = True │
│ ❱ 887 │ │ return call._call_and_handle_interrupt( │
│ 888 │ │ │ self, self._predict_impl, model, dataloaders, datamodule, │
│ 889 │ │ ) │
│ 890 │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/trainer/call.py:48 in │
│ _call_and_handle_interrupt │
│ │
│ 45 │ try: │
│ 46 │ │ if trainer.strategy.launcher is not None: │
│ 47 │ │ │ return trainer.strategy.launcher.launch(trainer_fn, *args, │
│ ❱ 48 │ │ return trainer_fn(*args, **kwargs) │
│ 49 │ │
│ 50 │ except _TunerExitException: │
│ 51 │ │ _call_teardown_hook(trainer) │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py:928 in │
│ _predict_impl │
│ │
│ 925 │ │ ckpt_path = self._checkpoint_connector._select_ckpt_path( │
│ 926 │ │ │ self.state.fn, ckpt_path, model_provided=model_provided, │
│ 927 │ │ ) │
│ ❱ 928 │ │ results = self._run(model, ckpt_path=ckpt_path) │
│ 929 │ │ │
│ 930 │ │ assert self.state.stopped │
│ 931 │ │ self.predicting = False │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py:1012 in │
│ _run │
│ │
│ 1009 │ │ # ---------------------------- │
│ 1010 │ │ # RUN THE TRAINER │
│ 1011 │ │ # ---------------------------- │
│ ❱ 1012 │ │ results = self._run_stage() │
│ 1013 │ │ │
│ 1014 │ │ # ---------------------------- │
│ 1015 │ │ # POST-Training CLEAN UP │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/trainer/trainer.py:1051 in │
│ _run_stage │
│ │
│ 1048 │ │ if self.evaluating: │
│ 1049 │ │ │ return self._evaluation_loop.run() │
│ 1050 │ │ if self.predicting: │
│ ❱ 1051 │ │ │ return self.predict_loop.run() │
│ 1052 │ │ if self.training: │
│ 1053 │ │ │ with isolate_rng(): │
│ 1054 │ │ │ │ self._run_sanity_check() │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/loops/utilities.py:179 in │
│ _decorator │
│ │
│ 176 │ │ else: │
│ 177 │ │ │ context_manager = torch.no_grad │
│ 178 │ │ with context_manager(): │
│ ❱ 179 │ │ │ return loop_run(self, *args, **kwargs) │
│ 180 │ │
│ 181 │ return _decorator │
│ 182 │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/loops/prediction_loop.py:122 │
│ in run │
│ │
│ 119 │ │ │ │ │ dataloader_idx = data_fetcher._dataloader_idx │
│ 120 │ │ │ │ else: │
│ 121 │ │ │ │ │ dataloader_iter = None │
│ ❱ 122 │ │ │ │ │ batch, batch_idx, dataloader_idx = next(data_fetch │
│ 123 │ │ │ │ self.batch_progress.is_last_batch = data_fetcher.done │
│ 124 │ │ │ │ # run step hooks │
│ 125 │ │ │ │ self._predict_step(batch, batch_idx, dataloader_idx, d │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/loops/fetchers.py:134 in │
│ __next__ │
│ │
│ 131 │ │ │ │ self.done = not self.batches │
│ 132 │ │ elif not self.done: │
│ 133 │ │ │ # this will run only when no pre-fetching was done. │
│ ❱ 134 │ │ │ batch = super().__next__() │
│ 135 │ │ else: │
│ 136 │ │ │ # the iterator is empty │
│ 137 │ │ │ raise StopIteration │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/loops/fetchers.py:61 in │
│ __next__ │
│ │
│ 58 │ │ assert self.iterator is not None │
│ 59 │ │ self._start_profiler() │
│ 60 │ │ try: │
│ ❱ 61 │ │ │ batch = next(self.iterator) │
│ 62 │ │ except StopIteration: │
│ 63 │ │ │ self.done = True │
│ 64 │ │ │ raise │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/utilities/combined_loader.py:3 │
│ 41 in __next__ │
│ │
│ 338 │ │
│ 339 │ def __next__(self) -> _ITERATOR_RETURN: │
│ 340 │ │ assert self._iterator is not None │
│ ❱ 341 │ │ out = next(self._iterator) │
│ 342 │ │ if isinstance(self._iterator, _Sequential): │
│ 343 │ │ │ return out │
│ 344 │ │ out, batch_idx, dataloader_idx = out │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/pytorch_lightning/utilities/combined_loader.py:1 │
│ 42 in __next__ │
│ │
│ 139 │ │ │ │ │ raise StopIteration │
│ 140 │ │ │
│ 141 │ │ try: │
│ ❱ 142 │ │ │ out = next(self.iterators[0]) │
│ 143 │ │ except StopIteration: │
│ 144 │ │ │ # try the next iterator │
│ 145 │ │ │ self._use_next_iterator() │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/torch/utils/data/dataloader.py:631 in __next__ │
│ │
│ 628 │ │ │ if self._sampler_iter is None: │
│ 629 │ │ │ │ # TODO(https://github.com/pytorch/pytorch/issues/7675 │
│ 630 │ │ │ │ self._reset() # type: ignore[call-arg] │
│ ❱ 631 │ │ │ data = self._next_data() │
│ 632 │ │ │ self._num_yielded += 1 │
│ 633 │ │ │ if self._dataset_kind == _DatasetKind.Iterable and \ │
│ 634 │ │ │ │ │ self._IterableDataset_len_called is not None and │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/torch/utils/data/dataloader.py:1329 in │
│ _next_data │
│ │
│ 1326 │ │ │ │ return self._process_data(data) │
│ 1327 │ │ │ │
│ 1328 │ │ │ assert not self._shutdown and self._tasks_outstanding > 0 │
│ ❱ 1329 │ │ │ idx, data = self._get_data() │
│ 1330 │ │ │ self._tasks_outstanding -= 1 │
│ 1331 │ │ │ if self._dataset_kind == _DatasetKind.Iterable: │
│ 1332 │ │ │ │ # Check for _IterableDatasetStopIteration │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/torch/utils/data/dataloader.py:1295 in _get_data │
│ │
│ 1292 │ │ │ # need to call `.task_done()` because we don't use `.join │
│ 1293 │ │ else: │
│ 1294 │ │ │ while True: │
│ ❱ 1295 │ │ │ │ success, data = self._try_get_data() │
│ 1296 │ │ │ │ if success: │
│ 1297 │ │ │ │ │ return data │
│ 1298 │
│ │
│ /gpfs/fs5/nrc/nrc-fs1/ict/others/u/tes001/miniforge3/envs/EveryVoice_pr606/l │
│ ib/python3.12/site-packages/torch/utils/data/dataloader.py:1160 in │
│ _try_get_data │
│ │
│ 1157 │ │ │ │ fs = [tempfile.NamedTemporaryFile() for i in range(fd │
│ 1158 │ │ │ except OSError as e: │
│ 1159 │ │ │ │ if e.errno == errno.EMFILE: │
│ ❱ 1160 │ │ │ │ │ raise RuntimeError( │
│ 1161 │ │ │ │ │ │ "Too many open files. Communication with the" │
│ 1162 │ │ │ │ │ │ " workers is no longer possible. Please incre │
│ 1163 │ │ │ │ │ │ " limit using `ulimit -n` in the shell or cha │
╰──────────────────────────────────────────────────────────────────────────────╯
RuntimeError: Too many open files. Communication with the workers is no longer
possible. Please increase the limit using `ulimit -n` in the shell or change the
sharing strategy by calling
`torch.multiprocessing.set_sharing_strategy('file_system')` at the beginning of
your code
```

_Originally posted by @marctessier in https://github.com/EveryVoiceTTS/EveryVoice/issues/606#issuecomment-2956926624_

Start with everyvoice/model/feature_prediction/FastSpeech2_lightning/fs2/cli/check_data.py, especially check_data_command, and trace its call into synthesize.py and trainer.predict. Reproduce everyvoice check-data with varying dataset sizes and CPU worker counts; done means the command completes without the reported too-many-open-files RuntimeError.

Written by the indexing model from the issue text.