sc.tl.embedding_density errors when a category has one observation

/usr/local/lib/python3.8/site-packages/scipy/stats/kde.py:563: RuntimeWarning: Degrees of freedom <= 0 for slice
  self._data_covariance = atleast_2d(cov(self.dataset, rowvar=1,
/usr/local/lib/python3.8/site-packages/numpy/lib/function_base.py:2493: RuntimeWarning: divide by zero encountered in true_divide
  c *= np.true_divide(1, fact)
/usr/local/lib/python3.8/site-packages/numpy/lib/function_base.py:2493: RuntimeWarning: invalid value encountered in multiply
  c *= np.true_divide(1, fact)
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/var/folders/bd/43q20k0n6z15tdfzxvd22r7c0000gn/T/ipykernel_4569/1429565136.py in <module>
----> 1 sc.tl.embedding_density(adata, groupby="annotation_broad_extrapolated")

~/github/scanpy/scanpy/tools/_embedding_density.py in embedding_density(adata, basis, groupby, key_added, components)
    164             embed_y = adata.obsm[f'X_{basis}'][cat_mask, components[1]]
    165 
--> 166             dens_embed = _calc_density(embed_x, embed_y)
    167             density_values[cat_mask] = dens_embed
    168 

~/github/scanpy/scanpy/tools/_embedding_density.py in _calc_density(x, y)
     19     # Calculate the point density
     20     xy = np.vstack([x, y])
---> 21     z = gaussian_kde(xy)(xy)
     22 
     23     min_z = np.min(z)

/usr/local/lib/python3.8/site-packages/scipy/stats/kde.py in __init__(self, dataset, bw_method, weights)
    204             self._neff = 1/sum(self._weights**2)
    205 
--> 206         self.set_bandwidth(bw_method=bw_method)
    207 
    208     def evaluate(self, points):

/usr/local/lib/python3.8/site-packages/scipy/stats/kde.py in set_bandwidth(self, bw_method)
    552             raise ValueError(msg)
    553 
--> 554         self._compute_covariance()
    555 
    556     def _compute_covariance(self):

/usr/local/lib/python3.8/site-packages/scipy/stats/kde.py in _compute_covariance(self)
    564                                                bias=False,
    565                                                aweights=self.weights))
--> 566             self._data_inv_cov = linalg.inv(self._data_covariance)
    567 
    568         self.covariance = self._data_covariance * self.factor**2

/usr/local/lib/python3.8/site-packages/scipy/linalg/basic.py in inv(a, overwrite_a, check_finite)
    937 
    938     """
--> 939     a1 = _asarray_validated(a, check_finite=
)
    940     if len(a1.shape) != 2 or a1.shape[0] != a1.shape[1]:
    941         raise ValueError('expected square matrix')

/usr/local/lib/python3.8/site-packages/scipy/_lib/_util.py in _asarray_validated(a, check_finite, sparse_ok, objects_ok, mask_ok, as_inexact)
    291             raise ValueError('masked arrays are not supported')
    292     toarray = np.asarray_chkfinite if check_finite else np.asarray
--> 293     a = toarray(a)
    294     if not objects_ok:
    295         if a.dtype is np.dtype('O'):

/usr/local/lib/python3.8/site-packages/numpy/lib/function_base.py in asarray_chkfinite(a, dtype, order)
    486     a = asarray(a, dtype=dtype, order=order)
    487     if a.dtype.char in typecodes['AllFloat'] and not np.isfinite(a).all():
--> 488         raise ValueError(
    489             "array must not contain infs or NaNs")
    490     return a

ValueError: array must not contain infs or NaNs