Score model locally failed, ValueError: Input contains NaN, infinity or a value too large for dtype('float32').

I am scoring a ResNet-50 model locally but run into this issue. This issue seems to only happen randomly to some models. I am scoring 30 models, and only 2 of the models have this issue; the rest of the models (containing models trained with the same condition but trained with different random seeds) could be scored well.

```
layer packaging: 89%|████████▉ | 17/19 [00:14<00:00, 3.02it/s]
layer packaging: 100%|██████████| 19/19 [00:14<00:00, 1.28it/s]
layer principal components: 0%| | 0/19 [00:00", line 1, in
File "/weka/scratch/weka/dicarlo/yu_xie/projects/multitask-vision/score_model.py", line 187, in prepare_and_score_model_slurm
prepare_and_score_model(config)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/multitask-vision/score_model.py", line 178, in prepare_and_score_model
score_model_on_a_benchmark(model, benchmark_id)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/multitask-vision/score_model.py", line 152, in score_model_on_a_benchmark
bscore = score_local_model(model=model, benchmark_identifier=benchmark)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/multitask-vision/score_model.py", line 82, in score_local_model
score = benchmark(model)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/benchmark_helpers/neural_common.py", line 27, in __call__
source_assembly = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/__init__.py", line 61, in look_at
return self.layer_model.look_at(stimuli, number_of_trials=number_of_trials)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/temporal.py", line 32, in look_at
responses = self._layer_model.look_at(stimuli, number_of_trials=number_of_trials)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/neural.py", line 29, in look_at
layers = self.region_layer_map[region]
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/__init__.py", line 83, in __getitem__
self.commit_region(region)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/__init__.py", line 88, in commit_region
best_layer = self.layer_selection(selection_identifier=region, benchmark=benchmark)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/neural.py", line 77, in __call__
benchmark=benchmark)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/result_caching/__init__.py", line 80, in wrapper
result = function(*args, **kwargs)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/neural.py", line 88, in _call
layers=self.layers, prerun=True)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/neural.py", line 109, in __call__
model=self._activations_model, benchmark=benchmark, layers=layers, prerun=prerun)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/result_caching/__init__.py", line 312, in wrapper
result = function(**reduced_call_args)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/neural.py", line 123, in _call
score = benchmark(layer_model)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/utils/__init__.py", line 80, in __call__
return self.content(*args, **kwargs)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/benchmark_helpers/neural_common.py", line 27, in __call__
source_assembly = candidate.look_at(stimulus_set, number_of_trials=self._number_of_trials)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/brain_transformation/neural.py", line 144, in look_at
self._model(layers=self._layers, stimuli=stimuli)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/pytorch.py", line 41, in __call__
return self._extractor(*args, **kwargs)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 41, in __call__
return self.from_stimulus_set(stimulus_set=stimuli, layers=layers, stimuli_identifier=stimuli_identifier)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 55, in from_stimulus_set
activations = self.from_paths(stimuli_paths=stimuli_paths, layers=layers, stimuli_identifier=stimuli_identifier)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 73, in from_paths
activations = fnc(layers=layers, stimuli_paths=reduced_paths)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/result_caching/__init__.py", line 312, in wrapper
result = function(**reduced_call_args)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 79, in _from_paths_stored
return self._from_paths(layers=layers, stimuli_paths=stimuli_paths)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 85, in _from_paths
layer_activations = self._get_activations_batched(stimuli_paths, layers=layers, batch_size=self._batch_size)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 135, in _get_activations_batched
batch_activations = hook(batch_activations)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/pca.py", line 22, in __call__
self._ensure_initialized(batch_activations.keys())
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/pca.py", line 39, in _ensure_initialized
n_components=self._n_components)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/result_caching/__init__.py", line 223, in wrapper
result = function(**reduced_call_args)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/pca.py", line 69, in _pcas
multithread=os.getenv('MT_MULTITHREAD', '1') == '1')
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 247, in change_dict
results = map_fnc(apply_change, d.items())
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/multiprocessing/pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/multiprocessing/pool.py", line 657, in get
raise self._value
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/multiprocessing/pool.py", line 121, in worker
result = (True, func(*args, **kwds))
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/multiprocessing/pool.py", line 44, in mapstar
return list(map(*args))
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/core.py", line 244, in apply_change
values = change_function(values) if not keep_name else change_function(layer, values)
File "/weka/scratch/weka/dicarlo/yu_xie/projects/brain-score/brainscore_vision/model_helpers/activations/pca.py", line 64, in init_and_progress
pca.fit(activations)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/sklearn/decomposition/_pca.py", line 382, in fit
self._fit(X)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/sklearn/decomposition/_pca.py", line 431, in _fit
X, dtype=[np.float64, np.float32], ensure_2d=True, copy=self.copy
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/sklearn/base.py", line 566, in _validate_data
X = check_array(X, **check_params)
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/sklearn/utils/validation.py", line 800, in check_array
_assert_all_finite(array, allow_nan=force_all_finite == "allow-nan")
File "/om/user/yu_xie/miniconda3/envs/brainscore/lib/python3.7/site-packages/sklearn/utils/validation.py", line 116, in _assert_all_finite
type_err, msg_dtype if msg_dtype is not None else X.dtype
ValueError: Input contains NaN, infinity or a value too large for dtype('float32').
```