PicklingError numpy 1.20
Nobody has claimed this yet.
- Dominant language
- Python
- Stars
- 8.3k
- Forks
- 871
- Avg merge
- 1d 13h
- Merged PRs (30d)
- 5
Description
I am having the following issue using numpy 1.20 and umpa-learn.
See the following traceback.
```
PicklingError Traceback (most recent call last)
in
19
20 UMAP = reducer.fit_transform(article_cluster_data[[
---> 21 s for s in eans_embeddings.columns if "embedding" in s
22 ]])
23
/databricks/python/lib/python3.7/site-packages/umap/umap_.py in fit_transform(self, X, y)
2633 Local radii of data points in the embedding (log-transformed).
2634 """
-> 2635 self.fit(X, y)
2636 if self.transform_mode == "embedding":
2637 if self.output_dens:
/databricks/python/lib/python3.7/site-packages/umap/umap_.py in fit(self, X, y)
2571
2572 numba.set_num_threads(self._original_n_threads)
-> 2573 self._input_hash = joblib.hash(self._raw_data)
2574
2575 return self
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in hash(obj, hash_name, coerce_mmap)
265 else:
266 hasher = Hasher(hash_name=hash_name)
--> 267 return hasher.hash(obj)
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in hash(self, obj, return_digest)
66 def hash(self, obj, return_digest=True):
67 try:
---> 68 self.dump(obj)
69 except pickle.PicklingError as e:
70 e.args += ('PicklingError while hashing %r: %r' % (obj, e),)
/databricks/python/lib/python3.7/pickle.py in dump(self, obj)
435 if self.proto >= 4:
436 self.framer.start_framing()
--> 437 self.save(obj)
438 self.write(STOP)
439 self.framer.end_framing()
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
240 klass = obj.__class__
241 obj = (klass, ('HASHED', obj.descr))
--> 242 Hasher.save(self, obj)
243
244
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
92 cls = obj.__self__.__class__
93 obj = _MyHash(func_name, inst, cls)
---> 94 Pickler.save(self, obj)
95
96 def memoize(self, obj):
/databricks/python/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/databricks/python/lib/python3.7/pickle.py in save_tuple(self, obj)
772 if n <= 3 and self.proto >= 2:
773 for element in obj:
--> 774 save(element)
775 # Subtle. Same as in the big comment below.
776 if id(obj) in memo:
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
240 klass = obj.__class__
241 obj = (klass, ('HASHED', obj.descr))
--> 242 Hasher.save(self, obj)
243
244
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
92 cls = obj.__self__.__class__
93 obj = _MyHash(func_name, inst, cls)
---> 94 Pickler.save(self, obj)
95
96 def memoize(self, obj):
/databricks/python/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/databricks/python/lib/python3.7/pickle.py in save_tuple(self, obj)
787 write(MARK)
788 for element in obj:
--> 789 save(element)
790
791 if id(obj) in memo:
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
240 klass = obj.__class__
241 obj = (klass, ('HASHED', obj.descr))
--> 242 Hasher.save(self, obj)
243
244
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
92 cls = obj.__self__.__class__
93 obj = _MyHash(func_name, inst, cls)
---> 94 Pickler.save(self, obj)
95
96 def memoize(self, obj):
/databricks/python/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
502 f = self.dispatch.get(t)
503 if f is not None:
--> 504 f(self, obj) # Call unbound method with explicit self
505 return
506
/databricks/python/lib/python3.7/pickle.py in save_tuple(self, obj)
772 if n <= 3 and self.proto >= 2:
773 for element in obj:
--> 774 save(element)
775 # Subtle. Same as in the big comment below.
776 if id(obj) in memo:
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
240 klass = obj.__class__
241 obj = (klass, ('HASHED', obj.descr))
--> 242 Hasher.save(self, obj)
243
244
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save(self, obj)
92 cls = obj.__self__.__class__
93 obj = _MyHash(func_name, inst, cls)
---> 94 Pickler.save(self, obj)
95
96 def memoize(self, obj):
/databricks/python/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
516 issc = False
517 if issc:
--> 518 self.save_global(obj)
519 return
520
/databricks/python/lib/python3.7/site-packages/joblib/hashing.py in save_global(self, obj, name, pack)
115 Pickler.save_global(self, obj, **kwargs)
116 except pickle.PicklingError:
--> 117 Pickler.save_global(self, obj, **kwargs)
118 module = getattr(obj, "__module__", None)
119 if module == '__main__':
/databricks/python/lib/python3.7/pickle.py in save_global(self, obj, name)
958 raise PicklingError(
959 "Can't pickle %r: it's not found as %s.%s" %
--> 960 (obj, module_name, name)) from None
961 else:
962 if obj2 is not obj:
```
```
PicklingError: ("Can't pickle : it's not found as numpy.dtype[float32]", 'PicklingError while hashing array([[-0.3997416 , -0.19219466, -0.83981943, ..., -0.9273374 ,\n 1.4046632 , 0.30895016],\n [-0.04274601, -0.12016755, -0.53093857, ..., -0.9320015 ,\n 0.8004919 , 0.14586882],\n [ 0.10363793, 0.21220148, -0.5180615 , ..., -1.103286 ,\n 1.030384 , 0.33772892],\n ...,\n [ 0.45876223, 0.13564155, -0.37127146, ..., -0.24023826,\n 0.6981608 , 0.5868731 ],\n [-0.12448474, -0.12088505, -0.5615971 , ..., -0.42116365,\n 1.4583211 , 0.395956 ],\n [-0.10243232, -0.24882779, 0.15550528, ..., -0.7924694 ,\n 1.1544111 , 0.19003616]], dtype=float32): PicklingError("Can\'t pickle : it\'s not found as numpy.dtype[float32]")')
```
Contributor guide
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Start in umap/umap_.py at fit_transform and fit, where joblib.hash is called on the input data, and compare that path with the provided NumPy 1.20 and Python 3.7 traceback. Reproduce the failure using the shown embedding-shaped input; done means fit_transform completes without the PicklingError.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- numpy, python
- Domain
- machine-learning
- Issue type
- Bug
- Difficulty
- 3/5
- Estimated time
- 1-2 days
- Activity status
- Stale
- Clarity
- Mostly clear
- Newbie friendliness
- 30/100