intake / intake/intake-thredds

issues opening GFS archive

Open
#26 3 comments 0 reactions 0 assignees View on GitHub
Dominant language
Python
Stars
11
Forks
7
PR merge metrics
No merged PRs in 30d

Description

Thanks for this packages. I tried this today but couldn't open a file. I'm able to open it use `xr.open_dataset` and i'm not sure if intake-thredds wants the data to be in a certain format.

I'm reading in archive GFS forecast (https://rda.ucar.edu/datasets/ds084.1/#!description). Note you will have to get a log in to access it (https://stackoverflow.com/questions/66178846/read-in-authorized-opendap-url-using-xarray)

```
url = "https://rda.ucar.edu/thredds/dodsC/files/g/ds084.1/2020/20200201/gfs.0p25.2020020100.f000.grib2"
ds = xr.open_mfdataset([url])
```

vs

```
cat_url = "https://rda.ucar.edu/thredds/catalog/files/g/ds084.1/2020/20200201/catalog.xml"
catalog = intake.open_thredds_cat(cat_url, name="GFS-catalog")
file = list(catalog)[0]
source = catalog[file]
ds = source().to_dask()

---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
in
----> 1 ds = source().to_dask()
2 ds

~/miniconda/envs/main/lib/python3.8/site-packages/intake_xarray/base.py in to_dask(self)
67 def to_dask(self):
68 """Return xarray object where variables are dask arrays"""
---> 69 return self.read_chunked()
70
71 def close(self):

~/miniconda/envs/main/lib/python3.8/site-packages/intake_xarray/base.py in read_chunked(self)
42 def read_chunked(self):
43 """Return xarray object (which will have chunks)"""
---> 44 self._load_metadata()
45 return self._ds
46

~/miniconda/envs/main/lib/python3.8/site-packages/intake/source/base.py in _load_metadata(self)
234 """load metadata only if needed"""
235 if self._schema is None:
--> 236 self._schema = self._get_schema()
237 self.dtype = self._schema.dtype
238 self.shape = self._schema.shape

~/miniconda/envs/main/lib/python3.8/site-packages/intake_xarray/base.py in _get_schema(self)
16
17 if self._ds is None:
---> 18 self._open_dataset()
19
20 metadata = {

~/miniconda/envs/main/lib/python3.8/site-packages/intake_xarray/opendap.py in _open_dataset(self)
92 import xarray as xr
93 store = self._get_store()
---> 94 self._ds = xr.open_dataset(store, chunks=self.chunks, **self._kwargs)

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/backends/api.py in open_dataset(filename_or_obj, group, decode_cf, mask_and_scale, decode_times, concat_characters, decode_coords, engine, chunks, lock, cache, drop_variables, backend_kwargs, use_cftime, decode_timedelta)
555
556 with close_on_error(store):
--> 557 ds = maybe_decode_store(store, chunks)
558
559 # Ensure source filename always stored in dataset object (GH issue #2550)

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/backends/api.py in maybe_decode_store(store, chunks)
451
452 def maybe_decode_store(store, chunks):
--> 453 ds = conventions.decode_cf(
454 store,
455 mask_and_scale=mask_and_scale,

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/conventions.py in decode_cf(obj, concat_characters, mask_and_scale, decode_times, decode_coords, drop_variables, use_cftime, decode_timedelta)
637 encoding = obj.encoding
638 elif isinstance(obj, AbstractDataStore):
--> 639 vars, attrs = obj.load()
640 extra_coords = set()
641 close = obj.close

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/backends/common.py in load(self)
111 """
112 variables = FrozenDict(
--> 113 (_decode_variable_name(k), v) for k, v in self.get_variables().items()
114 )
115 attributes = FrozenDict(self.get_attrs())

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/backends/pydap_.py in get_variables(self)
97
98 def get_variables(self):
---> 99 return FrozenDict(
100 (k, self.open_store_variable(self.ds[k])) for k in self.ds.keys()
101 )

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/core/utils.py in FrozenDict(*args, **kwargs)
451
452 def FrozenDict(*args, **kwargs) -> Frozen:
--> 453 return Frozen(dict(*args, **kwargs))
454
455

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/backends/pydap_.py in (.0)
98 def get_variables(self):
99 return FrozenDict(
--> 100 (k, self.open_store_variable(self.ds[k])) for k in self.ds.keys()
101 )
102

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/backends/pydap_.py in open_store_variable(self, var)
94 def open_store_variable(self, var):
95 data = indexing.LazilyOuterIndexedArray(PydapArrayWrapper(var))
---> 96 return Variable(var.dimensions, data, _fix_attributes(var.attributes))
97
98 def get_variables(self):

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/core/variable.py in __init__(self, dims, data, attrs, encoding, fastpath)
340 """
341 self._data = as_compatible_data(data, fastpath=fastpath)
--> 342 self._dims = self._parse_dimensions(dims)
343 self._attrs = None
344 self._encoding = None

~/miniconda/envs/main/lib/python3.8/site-packages/xarray/core/variable.py in _parse_dimensions(self, dims)
601 dims = tuple(dims)
602 if len(dims) != self.ndim:
--> 603 raise ValueError(
604 "dimensions %s must have the same length as the "
605 "number of data dimensions, ndim=%s" % (dims, self.ndim)

ValueError: dimensions ('height_above_ground_layer',) must have the same length as the number of data dimensions, ndim=2
```

Contributor guide

Open the contributing guide

Research direction

Reproduce the failure using the catalog URL and source().to_dask(), then read the intake_xarray/opendap.py traceback path and xarray's pydap backend call that raises the dimension mismatch. Compare it with the working xr.open_mfdataset example; done means the catalog source can open this GFS archive without the ValueError.

Written by the indexing model from the issue text.

Assessment

Tech stack
python
Domain
data
Issue type
Bug
Difficulty
3/5
Estimated time
1-2 days
Activity status
Stale
Clarity
Mostly clear
Newbie friendliness
35/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.