issues

It is not uncommon to find datasets with bad metadata, like gregorian_proleptic instead of the expected prolepic_gregorian [1], that will prevent users from reading the full dataset. Ideally we could have a functionality similar to iris' callbacks [2] to workaround this. In fact, it looks like xarray already does something similar in the open_mfdataset but not for the open_dataset.

Pinging @dcherian who gave the idea of using preprocess and @rsignell-usgs who is a pro on finding bad metadata everywhere.

[1] https://nbviewer.jupyter.org/gist/rsignell-usgs/27ba1fdeb934d6fd5b83abe43098a047

[2] https://scitools.org.uk/iris/docs/latest/userguide/navigating_a_cube.html?highlight=callback#adding-and-removing-metadata-to-the-cube-at-load-time

Not sure if the dataset is pathological or if the problem is in xarray. netCDF4 1.2.4 correctly returns dtype S1 but xarray 0.9.6 returns 'float64' and then fails to open the dataset. (I am also having issues loading this variable with netCDF4 >1.2.4.)

```python In [1]: import xarray as xr from netCDF4 import Dataset url = 'http://geoport.whoi.edu/thredds/dodsC/usgs/vault0/models/tides/vdatum_gulf_of_maine/adcirc54_38_orig.nc' nc = Dataset(url) ds = xr.open_dataset(url)

In [2]: nc.variables['tidenames'].dtype Out[2]: dtype('S1')

In [3]: ds['tidenames'].dtype Out[3]: dtype('float64')

In [4]: ds['tidenames'] Out[4]: --------------------------------------------------------------------------- ValueError Traceback (most recent call last) lib/python3.6/site-packages/IPython/core/formatters.py in call(self, obj) 691 type_pprinters=self.type_printers, 692 deferred_pprinters=self.deferred_printers) --> 693 printer.pretty(obj) 694 printer.flush() 695 return stream.getvalue()

lib/python3.6/site-packages/IPython/lib/pretty.py in pretty(self, obj) 378 if callable(meth): 379 return meth(obj, self, cycle) --> 380 return _default_pprint(obj, self, cycle) 381 finally: 382 self.end_group()

lib/python3.6/site-packages/IPython/lib/pretty.py in default_pprint(obj, p, cycle) 493 if _safe_getattr(klass, '__repr__', None) is not object.__repr__: 494 # A user-provided repr. Find newlines and replace them with p.break() --> 495 _repr_pprint(obj, p, cycle) 496 return 497 p.begin_group(1, '<')

lib/python3.6/site-packages/IPython/lib/pretty.py in repr_pprint(obj, p, cycle) 691 """A pprint that just redirects to the normal repr function.""" 692 # Find newlines and replace them with p.break() --> 693 output = repr(obj) 694 for idx,output_line in enumerate(output.splitlines()): 695 if idx:

lib/python3.6/site-packages/xarray/core/common.py in repr(self) 95 96 def repr(self): ---> 97 return formatting.array_repr(self) 98 99 def _iter(self):

lib/python3.6/site-packages/xarray/core/formatting.py in array_repr(arr) 384 summary.append(repr(arr.data)) 385 elif arr._in_memory or arr.size < 1e5: --> 386 summary.append(short_array_repr(arr.values)) 387 else: 388 summary.append(u'[%s values with dtype=%s]' % (arr.size, arr.dtype))

lib/python3.6/site-packages/xarray/core/dataarray.py in values(self) 401 def values(self): 402 """The array's data as a numpy.ndarray""" --> 403 return self.variable.values 404 405 @values.setter

lib/python3.6/site-packages/xarray/core/variable.py in values(self) 327 def values(self): 328 """The variable's data as a numpy.ndarray""" --> 329 return _as_array_or_item(self._data) 330 331 @values.setter

lib/python3.6/site-packages/xarray/core/variable.py in _as_array_or_item(data) 203 TODO: remove this (replace with np.asarray) once these issues are fixed 204 """ --> 205 data = np.asarray(data) 206 if data.ndim == 0: 207 if data.dtype.kind == 'M':

lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order) 480 481 """ --> 482 return array(a, dtype, copy=False, order=order) 483 484 def asanyarray(a, dtype=None, order=None):

lib/python3.6/site-packages/xarray/core/indexing.py in array(self, dtype) 425 426 def array(self, dtype=None): --> 427 self._ensure_cached() 428 return np.asarray(self.array, dtype=dtype) 429

lib/python3.6/site-packages/xarray/core/indexing.py in _ensure_cached(self) 422 def _ensure_cached(self): 423 if not isinstance(self.array, np.ndarray): --> 424 self.array = np.asarray(self.array) 425 426 def array(self, dtype=None):

lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order) 480 481 """ --> 482 return array(a, dtype, copy=False, order=order) 483 484 def asanyarray(a, dtype=None, order=None):

lib/python3.6/site-packages/xarray/core/indexing.py in array(self, dtype) 406 407 def array(self, dtype=None): --> 408 return np.asarray(self.array, dtype=dtype) 409 410 def getitem(self, key):

lib/python3.6/site-packages/numpy/core/numeric.py in asarray(a, dtype, order) 480 481 """ --> 482 return array(a, dtype, copy=False, order=order) 483 484 def asanyarray(a, dtype=None, order=None):

lib/python3.6/site-packages/xarray/core/indexing.py in array(self, dtype) 373 def array(self, dtype=None): 374 array = orthogonally_indexable(self.array) --> 375 return np.asarray(array[self.key], dtype=None) 376 377 def getitem(self, key):

lib/python3.6/site-packages/xarray/conventions.py in getitem(self, key) 365 def getitem(self, key): 366 return mask_and_scale(self.array[key], self.fill_value, --> 367 self.scale_factor, self.add_offset, self._dtype) 368 369 def repr(self):

lib/python3.6/site-packages/xarray/conventions.py in mask_and_scale(array, fill_value, scale_factor, add_offset, dtype) 61 """ 62 # by default, cast to float to ensure NaN is meaningful ---> 63 values = np.array(array, dtype=dtype, copy=True) 64 if fill_value is not None and not np.all(pd.isnull(fill_value)): 65 if getattr(fill_value, 'size', 1) > 1:

ValueError: could not convert string to float: 'STEADY ' ```

I will try to investigate this later this week.

Don't convert data with time units to timedeltas by default. Most of the time this behavior is not desirable (e.g: wave period data).

@shoyer suggest:

possibly we should add an explicit toggle for decoding timedeltas vs datetimes.

xref: https://github.com/pydata/xarray/pull/842#issuecomment-216933269