KeyError pulling from Nasa server with Pydap
Nobody has claimed this yet.
- Dominant language
- Python
- Stars
- 4.2k
- Forks
- 1.4k
- Avg merge
- 2d 15h
- Merged PRs (30d)
- 14
Description
What happened:
I'm trying to pull data from this NASA server: https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_H.002?. Through pydap, I can create a DataSet representing the data, but when I try to get the data I get this error:
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in _getitem_string(self, key)
403 try:
--> 404 return self._dict[quote(key)]
405 except KeyError:
KeyError: 'tmp2m%2Etmp2m'
During handling of the above exception, another exception occurred:
IndexError Traceback (most recent call last)
<ipython-input-17-3efbb8f7b71f> in <module>
----> 1 ds['tmp2m'].isel(time=0).values
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/dataarray.py in values(self)
632 def values(self) -> np.ndarray:
633 """The array's data as a numpy.ndarray"""
--> 634 return self.variable.values
635
636 @values.setter
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/variable.py in values(self)
552 def values(self):
553 """The variable's data as a numpy.ndarray"""
--> 554 return _as_array_or_item(self._data)
555
556 @values.setter
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/variable.py in _as_array_or_item(data)
285 data = data.get()
286 else:
--> 287 data = np.asarray(data)
288 if data.ndim == 0:
289 if data.dtype.kind == "M":
~/my-conda-envs/nwm/lib/python3.7/site-packages/numpy/core/_asarray.py in asarray(a, dtype, order, like)
100 return _asarray_with_like(a, dtype=dtype, order=order, like=like)
101
--> 102 return array(a, dtype, copy=False, order=order)
103
104
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/indexing.py in __array__(self, dtype)
691
692 def __array__(self, dtype=None):
--> 693 self._ensure_cached()
694 return np.asarray(self.array, dtype=dtype)
695
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/indexing.py in _ensure_cached(self)
688 def _ensure_cached(self):
689 if not isinstance(self.array, NumpyIndexingAdapter):
--> 690 self.array = NumpyIndexingAdapter(np.asarray(self.array))
691
692 def __array__(self, dtype=None):
~/my-conda-envs/nwm/lib/python3.7/site-packages/numpy/core/_asarray.py in asarray(a, dtype, order, like)
100 return _asarray_with_like(a, dtype=dtype, order=order, like=like)
101
--> 102 return array(a, dtype, copy=False, order=order)
103
104
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/indexing.py in __array__(self, dtype)
661
662 def __array__(self, dtype=None):
--> 663 return np.asarray(self.array, dtype=dtype)
664
665 def __getitem__(self, key):
~/my-conda-envs/nwm/lib/python3.7/site-packages/numpy/core/_asarray.py in asarray(a, dtype, order, like)
100 return _asarray_with_like(a, dtype=dtype, order=order, like=like)
101
--> 102 return array(a, dtype, copy=False, order=order)
103
104
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/indexing.py in __array__(self, dtype)
566 def __array__(self, dtype=None):
567 array = as_indexable(self.array)
--> 568 return np.asarray(array[self.key], dtype=None)
569
570 def transpose(self, order):
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/conventions.py in __getitem__(self, key)
60
61 def __getitem__(self, key):
---> 62 return np.asarray(self.array[key], dtype=self.dtype)
63
64
~/my-conda-envs/nwm/lib/python3.7/site-packages/numpy/core/_asarray.py in asarray(a, dtype, order, like)
100 return _asarray_with_like(a, dtype=dtype, order=order, like=like)
101
--> 102 return array(a, dtype, copy=False, order=order)
103
104
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/coding/variables.py in __array__(self, dtype)
68
69 def __array__(self, dtype=None):
---> 70 return self.func(self.array)
71
72 def __repr__(self):
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/coding/variables.py in _apply_mask(data, encoded_fill_values, decoded_fill_value, dtype)
136 ) -> np.ndarray:
137 """Mask all matching values in a NumPy arrays."""
--> 138 data = np.asarray(data, dtype=dtype)
139 condition = False
140 for fv in encoded_fill_values:
~/my-conda-envs/nwm/lib/python3.7/site-packages/numpy/core/_asarray.py in asarray(a, dtype, order, like)
100 return _asarray_with_like(a, dtype=dtype, order=order, like=like)
101
--> 102 return array(a, dtype, copy=False, order=order)
103
104
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/indexing.py in __array__(self, dtype)
566 def __array__(self, dtype=None):
567 array = as_indexable(self.array)
--> 568 return np.asarray(array[self.key], dtype=None)
569
570 def transpose(self, order):
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/backends/pydap_.py in __getitem__(self, key)
36 def __getitem__(self, key):
37 return indexing.explicit_indexing_adapter(
---> 38 key, self.shape, indexing.IndexingSupport.BASIC, self._getitem
39 )
40
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/core/indexing.py in explicit_indexing_adapter(key, shape, indexing_support, raw_indexing_method)
851 """
852 raw_key, numpy_indices = decompose_indexer(key, shape, indexing_support)
--> 853 result = raw_indexing_method(raw_key.tuple)
854 if numpy_indices.tuple:
855 # index the loaded np.ndarray
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/backends/pydap_.py in _getitem(self, key)
43 # downloading coordinate data twice
44 array = getattr(self.array, "array", self.array)
---> 45 result = robust_getitem(array, key, catch=ValueError)
46 # in some cases, pydap doesn't squeeze axes automatically like numpy
47 axis = tuple(n for n, k in enumerate(key) if isinstance(k, integer_types))
~/my-conda-envs/nwm/lib/python3.7/site-packages/xarray/backends/common.py in robust_getitem(array, key, catch, max_retries, initial_delay)
51 for n in range(max_retries + 1):
52 try:
---> 53 return array[key]
54 except catch:
55 if n == max_retries:
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in __getitem__(self, index)
318 def __getitem__(self, index):
319 out = copy.copy(self)
--> 320 out.data = self._get_data_index(index)
321 return out
322
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in _get_data_index(self, index)
347 return np.vectorize(decode_np_strings)(self._data[index])
348 else:
--> 349 return self._data[index]
350
351 def _get_data(self):
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/handlers/dap.py in __getitem__(self, index)
147 dataset = build_dataset(dds)
148 dataset.data = unpack_data(BytesReader(data), dataset)
--> 149 return dataset[self.id].data
150
151 def __len__(self):
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in __getitem__(self, key)
423 def __getitem__(self, key):
424 if isinstance(key, string_types):
--> 425 return self._getitem_string(key)
426 elif (isinstance(key, tuple) and
427 all(isinstance(name, string_types)
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in _getitem_string(self, key)
407 if len(splitted) > 1:
408 try:
--> 409 return self[splitted[0]]['.'.join(splitted[1:])]
410 except KeyError:
411 return self['.'.join(splitted[1:])]
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in __getitem__(self, index)
318 def __getitem__(self, index):
319 out = copy.copy(self)
--> 320 out.data = self._get_data_index(index)
321 return out
322
~/my-conda-envs/nwm/lib/python3.7/site-packages/pydap/model.py in _get_data_index(self, index)
347 return np.vectorize(decode_np_strings)(self._data[index])
348 else:
--> 349 return self._data[index]
350
351 def _get_data(self):
IndexError: only integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices
What you expected to happen:
I should be able to select the data w/o error.
Minimal Complete Verifiable Example:
(a nasa username and password are required):
from pydap.client import open_url
from pydap.cas.urs import setup_session
import xarray as xr
base_url = "https://hydro1.gesdisc.eosdis.nasa.gov/dods/NLDAS_FORA0125_H.002?"
session = setup_session("USER", "PASSWORD", check_url=base_url)
store = xr.backends.PydapDataStore.open(base_url, session=session)
ds = xr.open_dataset(store)
ds['tmp2m'].isel(time=0, lat=0, lon=0).values
Anything else we need to know?:
Environment:
Output of xr.show_versions()
INSTALLED VERSIONS
commit: None
python: 3.7.10 | packaged by conda-forge | (default, Feb 19 2021, 16:07:37)
[GCC 9.3.0]
python-bits: 64
OS: Linux
OS-release: 4.14.219-164.354.amzn2.x86_64
machine: x86_64
processor: x86_64
byteorder: little
LC_ALL: C.UTF-8
LANG: C.UTF-8
LOCALE: en_US.UTF-8
libhdf5: 1.10.6
libnetcdf: 4.7.4
xarray: 0.17.0
pandas: 1.2.3
numpy: 1.20.1
scipy: 1.6.0
netCDF4: 1.5.6
pydap: installed
h5netcdf: None
h5py: None
Nio: None
zarr: 2.6.1
cftime: 1.4.1
nc_time_axis: None
PseudoNetCDF: None
rasterio: 1.2.1
cfgrib: None
iris: None
bottleneck: None
dask: 2021.02.0
distributed: 2021.02.0
matplotlib: 3.3.4
cartopy: None
seaborn: None
numbagg: None
pint: None
setuptools: 49.6.0.post20210108
pip: 21.0.1
conda: None
pytest: None
IPython: 7.21.0
sphinx: None
Contributor guide
First steps
- Read the whole issue, then the project's contributing guide.
- Comment on the issue to say you are picking it up — it saves two people doing the same work.
- Fork the repository and make your change on a branch.
- Open a pull request that references the issue number.
Research direction
Start with xarray/backends/pydap_.py and the PydapDataStore.open entry point, then reproduce the provided ds['tmp2m'].isel(time=0, lat=0, lon=0).values example against the NASA URL. Trace the request into the pydap model.py calls shown in the traceback; done means selecting and loading the variable completes without the reported KeyError or IndexError.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- python
- Domain
- backend, data
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Stale
- Clarity
- Clearly specified
- Newbie friendliness
- 35/100