After adding the py38-all-but-dask CI in #3919, it discovered a few backend issues: - zarr: - [x] open_zarr with chunks="auto" always tries to chunk, even if dask is not available (fixed in #3919) - [x] ZarrArrayWrapper.__getitem__ incorrectly passes the indexer's tuple attribute to _arrayize_vectorized_indexer (this only happens if dask is not available) (fixed in #3919) - [x] slice indexers with negative steps get transformed incorrectly if dask is not available https://github.com/pydata/xarray/pull/8674 - rasterio: - ~calling pickle.dumps on a Dataset object returned by open_rasterio fails because a non-serializable lock was used (if dask is installed, a serializable lock is used instead)~

Is your feature request related to a problem?

I frequently find myself computing a handful of variables of a dataset (typically coordinates) and assigning them back to the dataset, and wishing we had a method / function that allowed that.

Describe the solution you'd like

I'd imagine something like python ds.compute(variables=variable_names) but I'm undecided on whether that's a good idea (it might make .compute more complex?)

Describe alternatives you've considered

So far I've been using something like python ds.assign_coords({k: lambda ds: ds[k].compute() for k in variable_names}) ds.pipe(lambda ds: ds.merge(ds[variable_names].compute())) but both are not easy to type / understand (though having .merge take a callable would make this much easier). Also, the first option computes variables separately, which may not be ideal?

Additional context

No response

I tried to run where with chunked cupy arrays: python In [1]: import xarray as xr ...: import cupy ...: import dask.array as da ...: ...: arr = xr.DataArray(cupy.arange(4), dims="x") ...: mask = xr.DataArray(cupy.array([False, True, True, False]), dims="x") this works: python In [2]: arr.where(mask) Out[2]: <xarray.DataArray (x: 4)> array([nan, 1., 2., nan]) Dimensions without coordinates: x this fails: ```python In [4]: arr.chunk().where(mask).compute()

TypeError Traceback (most recent call last) Cell In[4], line 1 ----> 1 arr.chunk().where(mask).compute()

File ~/repos/xarray/xarray/core/dataarray.py:1095, in DataArray.compute(self, kwargs) 1076 """Manually trigger loading of this array's data from disk or a 1077 remote source into memory and return a new array. The original is 1078 left unaltered. (...) 1092 dask.compute 1093 """ 1094 new = self.copy(deep=False) -> 1095 return new.load(kwargs)

File ~/repos/xarray/xarray/core/dataarray.py:1069, in DataArray.load(self, kwargs) 1051 def load(self: T_DataArray, kwargs) -> T_DataArray: 1052 """Manually trigger loading of this array's data from disk or a 1053 remote source into memory and return this array. 1054 (...) 1067 dask.compute 1068 """ -> 1069 ds = self._to_temp_dataset().load(**kwargs) 1070 new = self._from_temp_dataset(ds) 1071 self._variable = new._variable

File ~/repos/xarray/xarray/core/dataset.py:752, in Dataset.load(self, kwargs) 749 import dask.array as da 751 # evaluate all the dask arrays simultaneously --> 752 evaluated_data = da.compute(*lazy_data.values(), kwargs) 754 for k, data in zip(lazy_data, evaluated_data): 755 self.variables[k].data = data

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/base.py:600, in compute(traverse, optimize_graph, scheduler, get, args, kwargs) 597 keys.append(x.dask_keys()) 598 postcomputes.append(x.dask_postcompute()) --> 600 results = schedule(dsk, keys, kwargs) 601 return repack([f(r, a) for r, (f, a) in zip(results, postcomputes)])

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/threaded.py:89, in get(dsk, keys, cache, num_workers, pool, kwargs) 86 elif isinstance(pool, multiprocessing.pool.Pool): 87 pool = MultiprocessingPoolExecutor(pool) ---> 89 results = get_async( 90 pool.submit, 91 pool._max_workers, 92 dsk, 93 keys, 94 cache=cache, 95 get_id=_thread_get_id, 96 pack_exception=pack_exception, 97 kwargs, 98 ) 100 # Cleanup pools associated to dead threads 101 with pools_lock:

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/local.py:511, in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs) 509 _execute_task(task, data) # Re-execute locally 510 else: --> 511 raise_exception(exc, tb) 512 res, worker_id = loads(res_info) 513 state["cache"][key] = res

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/local.py:319, in reraise(exc, tb) 317 if exc.traceback is not tb: 318 raise exc.with_traceback(tb) --> 319 raise exc

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/local.py:224, in execute_task(key, task_info, dumps, loads, get_id, pack_exception) 222 try: 223 task, data = loads(task_info) --> 224 result = _execute_task(task, data) 225 id = get_id() 226 result = dumps((result, id))

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/core.py:119, in _execute_task(arg, cache, dsk) 115 func, args = arg[0], arg[1:] 116 # Note: Don't assign the subtask results to a variable. numpy detects 117 # temporaries by their reference count and can execute certain 118 # operations in-place. --> 119 return func(*(_execute_task(a, cache) for a in args)) 120 elif not ishashable(arg): 121 return arg

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/optimization.py:990, in SubgraphCallable.call(self, *args) 988 if not len(args) == len(self.inkeys): 989 raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args))) --> 990 return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/core.py:149, in get(dsk, out, cache) 147 for key in toposort(dsk): 148 task = dsk[key] --> 149 result = _execute_task(task, cache) 150 cache[key] = result 151 result = _execute_task(out, cache)

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/dask/core.py:119, in _execute_task(arg, cache, dsk) 115 func, args = arg[0], arg[1:] 116 # Note: Don't assign the subtask results to a variable. numpy detects 117 # temporaries by their reference count and can execute certain 118 # operations in-place. --> 119 return func(*(_execute_task(a, cache) for a in args)) 120 elif not ishashable(arg): 121 return arg

File <array_function internals>:180, in where(args, *kwargs)

File cupy/_core/core.pyx:1723, in cupy._core.core._ndarray_base.array_function()

File ~/.local/opt/mambaforge/envs/xarray/lib/python3.10/site-packages/cupy/_sorting/search.py:211, in where(condition, x, y) 209 if fusion._is_fusing(): 210 return fusion._call_ufunc(_where_ufunc, condition, x, y) --> 211 return _where_ufunc(condition.astype('?'), x, y)

File cupy/_core/_kernel.pyx:1287, in cupy._core._kernel.ufunc.call()

File cupy/_core/_kernel.pyx:160, in cupy._core._kernel._preprocess_args()

File cupy/_core/_kernel.pyx:146, in cupy._core._kernel._preprocess_arg()

TypeError: Unsupported type <class 'numpy.ndarray'> this works again:python In [7]: arr.chunk().where(mask.chunk(), cupy.array(cupy.nan)).compute() Out[7]: <xarray.DataArray (x: 4)> array([nan, 1., 2., nan]) Dimensions without coordinates: x `` And other methods likefillna` show similar behavior.

I think the reason is that this: https://github.com/pydata/xarray/blob/d4db16699f30ad1dc3e6861601247abf4ac96567/xarray/core/duck_array_ops.py#L195 is not sufficient to detect cupy beneath other layers of duckarrays (most commonly dask, pint, or both). In this specific case we could extend the condition to also match chunked cupy arrays (like arr.cupy.is_cupy does, but using is_duck_dask_array), but this will still break for other duckarray layers or if dask is not involved, and we're also in the process of moving away from special-casing dask. So short of asking cupy to treat 0d arrays like scalars I'm not sure how to fix this.

cc @jacobtomlinson

The upstream-dev CI currently fails sometimes due to a segfault (the normal CI crashes, too, but since we use pytest-xdist we only get a message stating "worker x crashed").

I'm not sure why, and I can't reproduce locally, either. Given that dask's local scheduler is in the traceback and the failing test is test_open_mfdataset_manyfiles, I assume there's some issue with parallel disk access or the temporary file creation.

What is your issue?

We frequently get bug reports related to encoding that can usually be fixed by clearing it or by overriding it using the encoding parameter of the to_* methods, e.g. - #4224 - #4380 - #4655 - #5427 - #5490 - fsspec/kerchunk#130

There are also a few discussions with more background: - https://github.com/pydata/xarray/pull/5065#issuecomment-806154872 - https://github.com/pydata/xarray/issues/1614 - #5082 - #5336

We discussed this in the meeting yesterday and as far as I can remember agreed that the current default behavior is not ideal and decided to investigate #5336: a keep_encoding option, similar to keep_attrs, that would be True (propagate encoding) by default but will be changed to False (drop encoding on any operation) in the future.

cc @rabernat, @shoyer

The most recent runs of the type checking CI have started to fail with a segfault: /home/runner/work/_temp/dac0c060-b19a-435a-8063-bbc5b8ffbf24.sh: line 1: 2945 Segmentation fault (core dumped) python -m mypy --install-types --non-interactive --cobertura-xml-report mypy_report This seems to be due to the release of mypy=0.990.

7269 pinned mypy to mypy<0.990, which should be undone once we figured out how to fix that (most likely by waiting on the next release), and any complaints the new version has (see this workflow run).

From #4355

@max-sixty:

Stepping back on the contributing doc — I admit I haven't look at it in a while — I wonder whether we can slim it down a bit, for example by linking to other docs for generic tooling — I imagine we're unlikely to have the best docs on working with GH, for example. Or referencing our PR template rather than the (now out-of-date) PR checklist.

We could also add a docstring guide since the numpydoc guide does not cover every little detail (for example, default notation, type spec vs. type hint, space before the colon separating parameter names from types, no colon for parameters without types, etc.)

This is a (incomplete) checklist for #5816 to make it easier to find methods that are in need of examples and links to the narrative docs with further information (of course, changes to the docstrings of all other methods / functions part of the public API are also appreciated).

Good examples explicitly construct small xarray objects to make it easier to follow (e.g. use np.{ones,full,zeros} or the np.array constructor instead of np.random / loading from files) and show both input and output of the function.

Use sh pytest --doctest-modules xarray --ignore xarray/tests/ to verify the examples, or push to a PR to have the CI do it for you (note that you will have much quicker feedback locally though).

To easily generate the expected output install pytest-accept (docs) in your dev environment and then run pytest --doctest-modules FILE_NAME --accept || true

To link to other documentation pages we can use python :doc:`project:label` Description of the linked page where we can leave out project if we link to somewhere within xarray's documentation. To figure out the label, we can either look at the source, search the output of python -m sphinx.ext.intersphinx https://docs.xarray.dev/en/latest/objects.inv, or use sphobjinv (install from PyPI): sh sphobjinv search -su https://docs.xarray.dev/en/latest/ missing

Top-level functions: - [ ] get_options - [ ] decode_cf - [ ] polyval - [ ] unify_chunks - [ ] infer_freq - [ ] date_range

I/O: - [ ] load_dataarray - [ ] load_dataset - [ ] open_dataarray - [ ] open_dataset - [ ] open_mfdataset

Contents: - [ ] DataArray.assign_attrs, Dataset.assign_attrs - [ ] DataArray.expand_dims, Dataset.expand_dims - [ ] DataArray.drop_duplicates, Dataset.drop_duplicates - [ ] DataArray.drop_vars, Dataset.drop_vars - [ ] Dataset.drop_dims - [ ] DataArray.convert_calendar, Dataset.convert_calendar - [ ] DataArray.set_coords, Dataset.set_coords - [ ] DataArray.reset_coords, Dataset.reset_coords

Comparisons: - [ ] DataArray.equals, Dataset.equals - [ ] DataArray.identical, Dataset.identical - [ ] DataArray.broadcast_equals, Dataset.broadcast_equals

Dask: - [ ] DataArray.compute, Dataset.compute - [ ] DataArray.chunk, Dataset.chunk - [ ] DataArray.persist, Dataset.persist

Missing values: - [ ] DataArray.bfill, Dataset.bfill - [ ] DataArray.ffill, Dataset.ffill - [ ] DataArray.fillna, Dataset.fillna - [ ] DataArray.dropna, Dataset.dropna

Indexing: - [ ] DataArray.loc (no docstring at all - came up in https://github.com/pydata/xarray/discussions/7528#discussion-4858556) - [ ] DataArray.drop_isel - [ ] DataArray.drop_sel - [ ] DataArray.head, Dataset.head - [ ] DataArray.tail, Dataset.tail - [ ] DataArray.interp_like, Dataset.interp_like - [ ] DataArray.reindex_like, Dataset.reindex_like - [ ] Dataset.isel

Aggregations: - [ ] Dataset.argmax - [ ] Dataset.argmin - [ ] DataArray.cumsum, Dataset.cumsum (intermediate to advanced) - [ ] DataArray.cumprod, Dataset.cumprod (intermediate to advanced) - [ ] DataArray.reduce, Dataset.reduce

We currently have two upstream-dev PR CI: the old pipelines CI and the new github actions CI added together with the scheduled upstream-dev ("nightly") CI. Since we don't need both I think we should disable one of these, presumably the old pipelines CI.

There's an issue with the CI result, though: since github doesn't have a icon for "passed with issues", we have to choose between "passed" or "failed" as the CI result (neither of which is optimal).

The advantage of using "failed" is that a failure is easily visible, but often the total CI result on PRs is set to "failed" because we didn't get around to fixing bugs introduced by recent changes to dependencies (which is confusing for contributors).

In #4584 I switched the pipelines upstream-dev CI to "allowed failure" so we get a warning instead of a failure. However, github doesn't print the number of warnings in the summary line, which means that if the icon is green nobody checks the status and upstream-dev CI failures are easily missed.

Our new scheduled nightly CI improves the situation quite a bit since we automatically get a issue containing the failures, but that means we aren't able to catch these failures before actually merging. As pointed out in https://github.com/pydata/xarray/issues/4574#issuecomment-725795622 that might be acceptable, though.

If we still want to fix this, we could have the PR CI automatically add a comment to the PR, which would contain a summary of the failures but also state that these failures can be ignored as long as they get the approval of a maintainer. This would increase the noise on PRs, though.

pint's implementation of NEP-18 (see hgrecco/pint#905) is close enough so we can finally start working on the pint support (i.e. make the integration tests pass). This would be the list of tasks to get there: * integration tests: - [x] implement integration tests for DataArray, Dataset and top-level functions (#3238, #3447, #3493) - [x] add tests for Variable as discussed in #3493 (#3654) - [x] clean up the current tests (#3600) - [x] use the standard assert_identical and assert_allclose functions (#3611, #3643, #3654, #3706, #3975) - [x] clean up the TestVariable.test_pad tests * actually get xarray to support units: - [x] top-level functions (#3611) - [x] Variable (#3706) + rolling_window and identical need larger modifications - [x] DataArray (#3643) - [x] Dataset - [x] silence all the UnitStrippedWarnings in the testsuite (#4163) - [ ] try to get nanprod to work with quantities - [x] add support for per variable fill values (#4165) - [x] repr with units (#2773) - [ ] type hierarchy (e.g. for np.maximum(data_array, quantity) vs np.maximum(quantity, data_array)) (#3950) * update the documentation - [x] point to pint-xarray (see #4530) - [x] mention the requirement for UnitRegistry(force_ndarray=True) or UnitRegistry(force_ndarray_like=True) (see https://pint-xarray.readthedocs.io/en/stable/creation.html#attaching-units) - [x] list the known issues (see https://github.com/pydata/xarray/pull/3643#issue-354872657 and https://github.com/pydata/xarray/pull/3643#issuecomment-602225731) (#4530): + pandas (indexing) + bottleneck (bfill, ffill) + scipy (interp) + numbagg (rolling_exp) + numpy.lib.stride_tricks.as_strided: rolling + numpy.vectorize: interpolate_na - [x] ~update the install instructions (we can use standard conda / pip now)~ this should be done by pint-xarray

There's a few unreleased and potentially breaking changes in main, most importantly the index refactor and the new groupby using numpy-groupies and flox. During the meeting on Wednesday we decided to release a preview version to get feedback before releasing a full version, especially from those who don't run their tests against our main branch.

I am planning to create the pre-release tomorrow, but if there's any big changes that should be included please post here.

cc @TomNicholas

Edit: the version will be called 2022.05.0.dev0, which will ensure that e.g. pip will require the --pre flag to install it.

The editable install on RTD is reported to have the version 0.17.1.dev0+g835a53e6.d20210226 (which technically is correct but it would be better to have a clean version on tags).

This is not something I can reproduce with python -m pip install -e ., so it is either some RTD weirdness or happens because we get mamba to do the editable install for us.

We should try to get the version right.

The most recent runs of the mypy CI on main are failing with: xarray/core/dataset.py:6934: error: Incompatible types in assignment (expression has type "str", variable has type "Optional[Optional[Literal['Y', 'M', 'W', 'D', 'h', 'm', 's', 'ms', 'us', 'ns', 'ps', 'fs', 'as']]]") [assignment] I'm not sure what changed since the last pass, though.

What is your issue?

As I mentioned in the meeting today, we have a lot of features where the the exception group support from PEP654 (which is scheduled for python 3.11 and consists of the class and a syntax change) might be useful. For example, we might want to collect all errors raised by rename in a exception group instead of raising them one-by-one.

For python<=3.10 there's a backport that contains the class and a workaround for the new syntax.

When adding two DataArray objects with different coordinates to a Dataset, a MergeError is raised even though one of the conflicting coords is a subset of the other. Merging dimension coordinates works so I'd expect associated non-dimension coordinates to work, too.

This fails: ```python In [1]: import xarray as xr ...: import numpy as np

In [2]: a = np.linspace(0, 1, 10) ...: b = np.linspace(-1, 0, 12) ...: ...: x_a = np.arange(10) ...: x_b = np.arange(12) ...: ...: y_a = x_a * 1000 ...: y_b = x_b * 1000 ...: ...: arr1 = xr.DataArray(data=a, coords={"x": x_a, "y": ("x", y_a)}, dims="x") ...: arr2 = xr.DataArray(data=b, coords={"x": x_b, "y": ("x", y_b)}, dims="x") ...: ...: xr.Dataset({"a": arr1, "b": arr2}) ... MergeError: conflicting values for variable 'y' on objects to be combined. You can skip this check by specifying compat='override'. While this works:python In [3]: a = np.linspace(0, 1, 10) ...: b = np.linspace(-1, 0, 12) ...: ...: x_a = np.arange(10) ...: x_b = np.arange(12) ...: ...: y_a = x_a * 1000 ...: y_b = x_b * 1000 ...: ...: xr.Dataset({ ...: "a": xr.DataArray(data=a, coords={"x": x_a}, dims="x"), ...: "b": xr.DataArray(data=b, coords={"x": x_b}, dims="x"), ...: }) Out[3]: <xarray.Dataset> Dimensions: (x: 12) Coordinates: * x (x) int64 0 1 2 3 4 5 6 7 8 9 10 11 Data variables: a (x) float64 0.0 0.1111 0.2222 0.3333 0.4444 ... 0.8889 1.0 nan nan b (x) float64 -1.0 -0.9091 -0.8182 -0.7273 ... -0.1818 -0.09091 0.0 ```

I can work around this by calling: python In [4]: xr.merge([arr1.rename("a").to_dataset(), arr2.rename("b").to_dataset()]) Out[4]: <xarray.Dataset> Dimensions: (x: 12) Coordinates: * x (x) int64 0 1 2 3 4 5 6 7 8 9 10 11 y (x) float64 0.0 1e+03 2e+03 3e+03 ... 8e+03 9e+03 1e+04 1.1e+04 Data variables: a (x) float64 0.0 0.1111 0.2222 0.3333 0.4444 ... 0.8889 1.0 nan nan b (x) float64 -1.0 -0.9091 -0.8182 -0.7273 ... -0.1818 -0.09091 0.0 but I think the Dataset constructor should be capable of that, too.

The documentation warns that the universal functions in xarray.ufuncs should not be used unless compatibility with numpy < 1.13 is required.

Since we only support numpy >= 1.15: is it time to remove that (already deprecated) module?

Since there are also functions that are not true ufuncs (e.g. np.angle and np.median) and need __array_function__ (or something similar, see #3917), we could also keep those and just remove the ones that are dispatched using __array_ufunc__.

What happened?

reindex stopped propagating attrs (detected in xarray-contrib/pint-xarray#159).

As far as I can tell, the new reindexing code in Aligner does not handle attrs yet?

Minimal Complete Verifiable Example

```Python

before #5692

In [1]: import xarray as xr ...: ...: xr.set_options(keep_attrs=True) ...: ...: ds = xr.tutorial.open_dataset("air_temperature")

In [2]: ds.reindex(lat=range(10, 80, 5)).lat Out[2]: <xarray.DataArray 'lat' (lat: 14)> array([10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75]) Coordinates: * lat (lat) int64 10 15 20 25 30 35 40 45 50 55 60 65 70 75 Attributes: standard_name: latitude long_name: Latitude units: degrees_north axis: Y

In [3]: ds.reindex(lat=xr.DataArray(range(10, 80, 5), attrs={"attr": "value"}, dims="lat")).lat Out[3]: <xarray.DataArray 'lat' (lat: 14)> array([10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75]) Coordinates: * lat (lat) int64 10 15 20 25 30 35 40 45 50 55 60 65 70 75 Attributes: standard_name: latitude long_name: Latitude units: degrees_north axis: Y

after #5692

In [3]: import xarray as xr ...: ...: xr.set_options(keep_attrs=True) ...: ...: ds = xr.tutorial.open_dataset("air_temperature")

In [4]: ds.reindex(lat=range(10, 80, 5)).lat Out[4]: <xarray.DataArray 'lat' (lat: 14)> array([10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75]) Coordinates: * lat (lat) int64 10 15 20 25 30 35 40 45 50 55 60 65 70 75

In [5]: ds.reindex(lat=xr.DataArray(range(10, 80, 5), attrs={"attr": "value"}, dims="lat")).lat Out[5]: <xarray.DataArray 'lat' (lat: 14)> array([10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75]) Coordinates: * lat (lat) int64 10 15 20 25 30 35 40 45 50 55 60 65 70 75 ```

What happened?

scipy seems to have some trouble with their documentation host setup, which means that trying to fetch its intersphinx registry returns a 404.

There's nothing we can do to really fix this, but we can try to the avoid docs build failures by disabling that intersphinx entry (not sure if that results in other errors, though)

What did you expect to happen?

No response

Minimal Complete Verifiable Example

No response

Relevant log output

python WARNING: failed to reach any of the inventories with the following issues: intersphinx inventory 'https://docs.scipy.org/doc/scipy/objects.inv' not fetchable due to <class 'requests.exceptions.HTTPError'>: 404 Client Error: Not Found for url: https://docs.scipy.org/doc/scipy/objects.inv

Anything else we need to know?

No response

Environment

See RTD

The cancel step doesn't seem to be configured properly, a push to a pull request shouldn't cancel CI on master. For reference, here are the logs for two examples: CI Additional, CI.

Continuing from #3602, load_store and dump_to_store look like they are old and unmaintained functions: * load_store is referenced once in api.rst (I assume the reference to from_store was to load_store), but never tested, used or mentioned anywhere else * dump_to_store is tested (and probably used), but never mentioned except from the section on backends

what should we do with these? Are they obsolete and should be removed or just unmaintained (then we should properly document and test them).

From #4108:

I wonder if it would be better to first "reorganize" all of the existing functions: we currently have rename (and Dataset.rename_dims / Dataset.rename_vars), set_coords, reset_coords, set_index, reset_index and swap_dims, which overlap partially. For example, the code sample from #4417 works if instead of python ds = ds.rename(b='x') ds = ds.set_coords('x') we use python ds = ds.set_index(x="b") and something similar for the code sample in #4107.

I believe we currently have these use cases (not sure if that list is complete, though): - rename a DataArray → rename - rename a existing variable to a name that is not yet in the object → rename / Dataset.rename_vars / Dataset.rename_dims - convert a data variable to a coordinate (not a dimension coordinate) → set_coords - convert a coordinate (not a dimension coordinate) to a data variable → reset_coords - swap a existing dimension coordinate with a coordinate (which may not exist) and rename the dimension → swap_dims - use a existing coordinate / data variable as a dimension coordinate (do not rename the dimension) → set_index - stop using a coordinate as dimension coordinate and append _ to its name (do not rename the dimension) → reset_index - use two existing coordinates / data variables as a MultiIndex → set_index - stop using a MultiIndex as a dimension coordinate and use its levels as coordinates → reset_index

Sometimes, some of these can be emulated by combinations of others, for example: ```python

x is a dimension without coordinates

assert_identical(ds.set_index({"x": "b"}), ds.swap_dims({"x": "b"}).rename({"b": "x"})) assert_identical(ds.swap_dims({"x": "b"}), ds.set_index({"x": "b"}).rename({"x": "b"})) and, with this PR:python assert_identical(ds.set_index({"x": "b"}), ds.set_coords("b").rename({"b": "x"})) assert_identical(ds.swap_dims({"x": "b"}), ds.rename({"b": "x"})) `` which means that it would increase the overlap ofrename,set_index, andswap_dims`.

In any case I think we should add a guide which explains which method to pick in which situation (or extend howdoi).

Originally posted by @keewis in https://github.com/pydata/xarray/issues/4108#issuecomment-761907785

From https://github.com/xarray-contrib/pint-xarray/pull/61#discussion_r662485351

matplotlib has a module called matplotlib.units which manages a mapping of types to hooks. This is then used to convert custom types to something matplotlib can work with, and to optionally add axis labels. For example, with pint: python In [9]: ureg = pint.UnitRegistry() ...: ureg.setup_matplotlib() ...: ...: t = ureg.Quantity(np.arange(10), "s") ...: v = ureg.Quantity(5, "m / s") ...: x = v * t ...: ...: fig, ax = plt.subplots(1, 1) ...: ax.plot(t, x) ...: ...: plt.show() this will plot the data without UnitStrippedWarnings and even attach the units as labels to the axis (the format is hard-coded in pint right now).

While this is pretty neat there are some issues: - xarray's plotting code converts to masked_array, dropping metadata on the duck array (which means matplotlib won't see the duck arrays) - we will end up overwriting the axis labels once the variable names are added (not sure if there's a way to specify a label format?) - it is matplotlib specific, which means we have to reimplement once we go through with the plotting entrypoints discussed in #3553 and #3640

All of this makes me wonder: should we try to maintain our own mapping of hooks which "prepare" the object based on the data's type? My initial idea would be that the hook function receives a Dataset or DataArray object and modifies it to convert the data to numpy arrays and optionally modifies the attrs.

For example for pint the hook would return the result of .pint.dequantify() but it could also be used to explicitly call .get on cupy arrays or .todense on sparse arrays.

xref #5561

In the pint integration tests I tried to also test calling numpy functions on xarray objects (we provide methods for all of them).

Some of these functions, like numpy.median, numpy.searchsorted and numpy.clip, depend on __array_function__ (i.e. not __array_ufunc__) to dispatch. However, neither Dataset nor DataArray (nor Variable, I think?) define these protocols (see #3643).

Should we define __array_function__ on xarray objects?

The pre-commit is reporting some mypy issues on the main branch: xarray/core/formatting.py:193: error: No overload variant of "max" matches argument types "signedinteger[Any]", "int" [call-overload] xarray/core/formatting.py:193: note: Possible overload variant: xarray/core/formatting.py:193: note: def [SupportsLessThanT <: SupportsLessThan] max(__arg1, SupportsLessThanT, SupportsLessThanT, *_args: SupportsLessThanT, key: None = ...) -> SupportsLessThanT xarray/core/formatting.py:193: note: <5 more non-matching overloads not shown> xarray/core/indexes.py:166: error: Incompatible types in assignment (expression has type "Union[dtype[object_], dtype[Any], dtype[void]]", variable has type "dtype[object_]") [assignment] xarray/core/computation.py:607: error: Argument 1 to "append" of "list" has incompatible type "None"; expected "slice" [arg-type] xarray/core/accessor_str.py:277: error: <nothing> not callable [misc] Found 4 errors in 4 files (checked 143 source files)

does anyone know how to fix those?

This was reported in #5077 and seems to be an upstream issue (see https://github.com/pydata/xarray/issues/5077#issuecomment-808605313):

FAILED xarray/tests/test_plot.py::TestCFDatetimePlot::test_cfdatetime_line_plot FAILED xarray/tests/test_plot.py::TestCFDatetimePlot::test_cfdatetime_pcolormesh_plot FAILED xarray/tests/test_plot.py::TestCFDatetimePlot::test_cfdatetime_contour_plot

In order to get the upsteam-dev CI open new issues for other failing tests on upstream-dev I've xfail'ed them in #5343. This should be undone once it is fixed in either cftime or nc-time-axis.

From xarray-contrib/xarray-tutorial#43 by @scottyhq:

Many notebooks out there start with the line ds = xr.tutorial.load_dataset("air_temperature"). That now gives an error traceback with xarray>=0.18:

pytb Traceback (most recent call last): File "/Users/scott/GitHub/zarrdata/./create_zarr.py", line 6, in <module> ds = xr.tutorial.load_dataset("air_temperature") File "/Users/scott/miniconda3/envs/zarrdata/lib/python3.9/site-packages/xarray/tutorial.py", line 179, in load_dataset with open_dataset(*args, **kwargs) as ds: File "/Users/scott/miniconda3/envs/zarrdata/lib/python3.9/site-packages/xarray/tutorial.py", line 100, in open_dataset ds = _open_dataset(filepath, **kws) File "/Users/scott/miniconda3/envs/zarrdata/lib/python3.9/site-packages/xarray/backends/api.py", line 485, in open_dataset engine = plugins.guess_engine(filename_or_obj) File "/Users/scott/miniconda3/envs/zarrdata/lib/python3.9/site-packages/xarray/backends/plugins.py", line 112, in guess_engine raise ValueError("cannot guess the engine, try passing one explicitly") ValueError: cannot guess the engine, try passing one explicitly

It's an easy fix though, just add ds = xr.tutorial.load_dataset("air_temperature", engine="netcdf4"), new users might be thrown by that though. Also a note that unless the netcdf4 library is explicitly put into the software environment, even adding the engine=netcdf4 can result in an error: "ValueError: unrecognized engine netcdf4 must be one of: ['store', 'zarr']", so I think a minimal environment definition to run would be: name: xarray-tutorial channels: - conda-forge dependencies: - xarray=0.18 - pooch=1.3 - netcdf4=1.5 - zarr=2.8

4906 added a autoupdate CI for the pre-commit hooks. However, once the hook versions are updated, the pre-commit CI is not run so potential errors will only be visible after merging the PR.

The documentation of create-pr-action states that that's because GITHUB_TOKEN does not allow it to trigger those workflows so we'd have to create a personal access token, or run pre-commit run --all-files immediately after the autoupdate.

The second option is definitely easier but would miss the separate mypy job (not sure how much of an issue that is).

It seems cfgrib (or one of its dependencies) is breaking the import of xarray on both upsteam-dev and some of the normal CI. Interestingly, the broken environments installed eccodes=2.19.1 while the ones which are passing have eccodes=2.21.0.

Is this a mamba / conda caching issue?

cc @andersy005 @alexamici

4248 added a _repr_inline_ method duck arrays can use to customize their collapsed variable repr.

We currently also have inline_dask_repr and inline_sparse_repr which remove redundant information like dtype and shape from dask and sparse arrays.

In order to reduce the complexity of inline_variable_array_repr, we could try to push these functions upstream.

While implementing the new _repr_inline_ in xarray-contrib/pint-xarray#22, I realized that I designed that method with a single level of nesting in mind, e.g. xarray(pint(x)) or xarray(dask(x)).

From that PR: @keewis

thinking about this some more, this doesn't work for anything other than numpy.ndarray objects. For now I guess we could use the magnitude's _repr_inline_ (falling back to __repr__ if that doesn't exist) and only use format_array_flat if the magnitude is a ndarray.

However, as we nest deeper (e.g. xarray(pint(uncertainties(dask(sparse(cupy))))) – for argument's sake, let's assume that this actually makes sense) this might break or become really complicated. Does anyone have any ideas how to deal with that?

If I'm simply missing something we have that discussion here, otherwise I guess we should open a issue on xarray's issue tracker.

@jthielen

Yes, I agree that format_array_flat should probably just apply to magnitude being an ndarray.

I think a cascading series of _repr_inline_ should work for nested arrays, so long as

* the metadata of the higher nested objects is considered the priority (if not, then we're back to a fully managed solution to the likes of [dask/dask#5329](https://github.com/dask/dask/issues/5329))

* small max lengths are handled gracefully (i.e., a minimum where it is just like `Dask.Array(...)`, then `...`, then nothing)

* we're okay with the lowest arrays in large nesting chains not having any information show up in the inline repr (situation where there is not enough characters to even describe the full nesting has to be accounted for somehow)

* it can be adopted without too much complaint across the ecosystem

Assuming all this, then each layer of the nesting will reduce the max length of the inline repr string available to the layers below it, until a layer reaches a reasonable minimum where it "gives up". At least that's the natural design that I inferred from the simple _repr_inline_(max_width) API.

All that being said, it might still be good to bring up on xarray's end since this is a more general issue with inline reprs of nested duck arrays, with nothing pint-specific other than it being the motivating use-case.

How should we deal with this?

the normal CI started to fail: FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[365_day] FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[360_day] FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[julian] FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[all_leap] FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[366_day] FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[gregorian] FAILED xarray/tests/test_cftimeindex.py::test_distant_cftime_datetime_sub_cftimeindex[proleptic_gregorian]

This seems to coincide with the release of pandas=1.2.4, and this is indeed the only difference in the environment (I didn't try to reproduce, though).

cc @spencerkclark

When running sphinx to build the documentation, it frequently times out when trying to build the visualization gallery. Running bash /usr/bin/time -v python -c 'import xarray as xr; xr.open_rasterio("https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif")' reports that it takes at least 5 minutes (or time out after 10 minutes) if opened from the url. Subsequent calls use the cache, so the second rasterio example is fast.

If instead I download the file manually and then load from disk, the whole notebook completes in about 10 seconds. Also, directly calling rasterio.open completes in a few seconds, so the bug should be in open_rasterio.

I do think we should try to fix this in the backend, but maybe we could also cache RGB.byte.tif in the same directory as the xarray.tutorial data and open the cached file in the gallery?

Edit: this is really flaky, I can't reliably reproduce this.

Edit2: for now, I'm using a extra cell containing ```python import pathlib import shutil import requests

cache_dir = pathlib.Path.home() / ".xarray_tutorial_data" path = cache_dir / "RGB.byte.tif" url = "https://github.com/mapbox/rasterio/raw/master/tests/data/RGB.byte.tif"

if not path.exists() or path.stat().st_size == 0: with requests.get(url) as r, path.open(mode="wb") as f: if r.status_code == requests.codes.ok: shutil.copyfileobj(r.raw, f) else: print("download failed: {r.status_code}") r.raise_for_status()

url = path ``` and modify both examples to use the new url

Recent changes in matplotlib (my guess is matplotlib/matplotlib#17830) seem to have broken a few of our tests:

It seems we have (seemingly) random failures on RTD.

Some of these are the known memory issue: recreating my doc environment used about 1.4 GB of RAM, which might be too much for RTD, even with the extended memory.

Much more often is a timeout when building the docs but I can't reproduce them locally. Any ideas? Edit: This really is random, I tried rerunning and the build passed.

Also, a warning: proj_create: init=epsg:/init=IGNF: syntax not supported in non-PROJ4 emulation mode

When writing accessors which behave the same for both Dataset and DataArray, it would be incredibly useful to be able to use DataArray._to_temp_dataset / DataArray._from_temp_dataset to deduplicate code. Is it safe to use those in external packages (like pint-xarray)?

Otherwise I guess it would be possible to use python name = da.name if da.name is None else "__temp" temp_ds = da.to_dataset(name=name) new_da = temp_ds[name] if da.name is None: new_da = new_da.rename(da.name) assert_identical(da, new_da) but that seems less efficient.

The scheduled CI is being skipped since about 6 days ago (which means this is probably due to the merge of #4729, see this run before the merge and this run after the merge).

This is really strange because workflow_dispatch events (for which the CI detection job is also skipped) still work perfectly fine.

Edit: it seems to be because of https://github.com/pydata/xarray/blob/f52a95cbe694336fe47bc5a42c713bee8ad74d64/.github/workflows/upstream-dev-ci.yaml#L34-L37 if I remove that the scheduled CI runs.

cc @andersy005

NEP 29 states that as of Jun 23, 2020 we can drop python 3.6 support. pandas will do so in their upcoming 1.2 release, so we might eventually be forced to do that, too.

It seems the scheduled CI failed because no artifacts were uploaded (i.e. the CI passed). Is it possible to skip the report job if the upstream-dev job passed, or to at least skip all following jobs after we found out that there are no artifacts to download? If not or if that turns out to be too complex we can just ignore the failure.

cc @andersy005

Continuing from #4567, trying to upgrade mypy to 0.790 fails with: xarray/core/utils.py:466: error: Value of type variable "_LT" of "sorted" cannot be "K" See also https://github.com/pydata/xarray/pull/4567#issuecomment-723546797 and https://github.com/pydata/xarray/pull/4567#issuecomment-724016427

In xarray/tests/test_dataset.py::TestDataset::test_polyfit_warnings, the second call to polyfit: python ds.var1.polyfit("dim2", 10, full=True)

emits a warning with the current upstream-dev pandas: xarray/tests/test_dataset.py::TestDataset::test_polyfit_warnings .../xarray/core/alignment.py:307: FutureWarning: Index.__or__ operating as a set operation is deprecated, in the future this will be a logical operation matching Series.__or__. Use index.union(other) instead index = joiner(matching_indexes)

since the test makes sure only the first call emits a warning (a np.RankWarning), the test fails.

With the merge of #3988 we're now properly documenting the str, dt and plot accessors, but the signatures of the plotting methods are missing a few parameters. For example, DataArray.plot.contour is missing the parameter x (it's properly documented in the parameters section, though).

Also, we need to remove the str and dt accessors from Computing and try to figure out how to fix the summary of DataArray.plot (the plotting method).

While working on #4163 I noticed that the fill value parameter for align (but maybe also reindex, concat, merge and combine_*?) will be used for all variables (except dimension coordinates) which is obviously not ideal when working with quantities. Would it make sense to optionally allow fill_value to be a dict which maps a fill value to a variable name?

Consider this: python In [2]: a = xr.Dataset( ...: data_vars={"a": ("x", [12, 14, 13, 10, 8])}, ...: coords={"x": [-2, -1, 0, 1, 2], "u": ("x", [-20, -10, 0, 10, 20])}, ...: ) ...: b = xr.Dataset( ...: data_vars={"b": ("x", [7, 9, 3])}, ...: coords={"x": [0, 3, 4], "u": ("x", [0, 30, 40])}, ...: ...: ) ...: ...: xr.align(a, b, join="outer", fill_value=-50) Out[2]: (<xarray.Dataset> Dimensions: (x: 7) Coordinates: * x (x) int64 -2 -1 0 1 2 3 4 u (x) int64 -20 -10 0 10 20 -50 -50 Data variables: a (x) int64 12 14 13 10 8 -50 -50, <xarray.Dataset> Dimensions: (x: 7) Coordinates: * x (x) int64 -2 -1 0 1 2 3 4 u (x) int64 -50 -50 0 -50 -50 30 40 Data variables: b (x) int64 -50 -50 7 -50 -50 9 3) I'd like to be able to do something like this instead: python In [3]: xr.align(a, b, join="outer", fill_value={"a": -30, "b": -40, "u": -50}) Out[3]: (<xarray.Dataset> Dimensions: (x: 7) Coordinates: * x (x) int64 -2 -1 0 1 2 3 4 u (x) int64 -20 -10 0 10 20 -50 -50 Data variables: a (x) int64 12 14 13 10 8 -30 -30, <xarray.Dataset> Dimensions: (x: 7) Coordinates: * x (x) int64 -2 -1 0 1 2 3 4 u (x) int64 -40 -40 0 -40 -40 30 40 Data variables: b (x) int64 -50 -50 7 -50 -50 9 3)

I could get there by passing the default (dtypes.NA) and then using fillna, but that only seems to work with data variables so coordinates would need to pass through a reset_coords / set_coords cycle. Also, with this the dtype is changed to float.

While working on #4286 I noticed that the docstring of DataArray.str.get claims to allow passing a default value in addition to the index, but the python code doesn't have that parameter at all. I think the default value is a good idea and that we should make the code match the docstring.

The RTD builds are timing out again. With my own setup I get this error instead:

I think that was introduced in #4219. cc @fujiisoup

Also, we should definitely ask support why those two behave differently. Edit: see readthedocs/readthedocs.org#7371

I'm not quite sure why (maybe a pandas or matplotlib release?), but the docs CI raises a exception in plotting.rst: https://github.com/pydata/xarray/blob/a081d01df11610adea7a48acee5a71d9eb5ffd16/doc/plotting.rst#L589-L590

there are also sphinx warnings about malformed rst: /home/docs/checkouts/readthedocs.org/user_builds/xray/conda/4286/lib/python3.8/site-packages/pandas/core/base.py:docstring of xarray.CFTimeIndex.max:7: WARNING: Inline strong start-string without end-string. /home/docs/checkouts/readthedocs.org/user_builds/xray/conda/4286/lib/python3.8/site-packages/pandas/core/base.py:docstring of xarray.CFTimeIndex.min:7: WARNING: Inline strong start-string without end-string. so I guess there was a pandas release a few days ago?

Edit: pandas 1.1.0 was released yesterday

Because I've been hit by this elsewhere: isort has released a new version today that removes the -rc / --recursive flag (without deprecation, I think). Once conda-forge has been updated, we will start seeing a failing isort CI.

Not sure if we should pin isort for now or simply remove the flag once the builds are failing (or we remove the flag now and have a pretty much useless isort CI until the feedstock has been merged).

Update: isort relaxed the error into a warning, so this is not that urgent

Due to the update of pytest, mypy reports errors: xarray/tests/test_cftimeindex_resample.py:57: error: List or tuple expected as variable arguments xarray/tests/test_dataset.py:2407: error: No overload variant of "__call__" of "_XfailMarkDecorator" matches argument type "Type[AssertionError]" xarray/tests/test_dataset.py:2407: note: Possible overload variant: xarray/tests/test_dataset.py:2407: note: def __call__(self, condition: Union[str, bool] = ..., *conditions: Union[str, bool], reason: str = ..., run: bool = ..., raises: Union[BaseException, Tuple[BaseException, ...]] = ..., strict: bool = ...) -> MarkDecorator xarray/tests/test_dataset.py:2407: note: <1 more non-matching overload not shown> xarray/tests/test_dask.py:767: error: No overload variant of "__call__" of "_XfailMarkDecorator" matches argument type "Type[NotImplementedError]" xarray/tests/test_dask.py:767: note: Possible overload variant: xarray/tests/test_dask.py:767: note: def __call__(self, condition: Union[str, bool] = ..., *conditions: Union[str, bool], reason: str = ..., run: bool = ..., raises: Union[BaseException, Tuple[BaseException, ...]] = ..., strict: bool = ...) -> MarkDecorator xarray/tests/test_dask.py:767: note: <1 more non-matching overload not shown> xarray/tests/test_dataarray.py:3953: error: No overload variant of "__call__" of "_XfailMarkDecorator" matches argument type "Type[AssertionError]" xarray/tests/test_dataarray.py:3953: note: Possible overload variant: xarray/tests/test_dataarray.py:3953: note: def __call__(self, condition: Union[str, bool] = ..., *conditions: Union[str, bool], reason: str = ..., run: bool = ..., raises: Union[BaseException, Tuple[BaseException, ...]] = ..., strict: bool = ...) -> MarkDecorator xarray/tests/test_dataarray.py:3953: note: <1 more non-matching overload not shown> Found 4 errors in 4 files (checked 124 source files) since that particular pytest version is a release candidate, should we pin pytest for now?

The upstream-dev CI currently fails because matplotlib upstream seems to have removed / renamed Colorbar._label. We definitely need to change that since it's a private attribute, but I'm not quite sure what we can use instead.

It seems numpy recently changed their dtype system and we now get warnings for np.bool, np.int and np.long. Instead, the warnings suggest we should use the python types or np.compat.long.

Since we have a few tests that require zero warnings, this means our test suite fails (and more than 13000 warnings are not a good thing, anyways).

At the moment, the output of a failing assert_allclose is not really useful, it simply prints the values it compared: ```python In [1]: import numpy as np ...: import xarray as xr ...: from xarray.testing import assert_allclose, assert_identical

In [2]: da1 = xr.DataArray(data=[2, 5, 8], coords={"x": [0, 1, 2]}, dims="x") ...: da2 = xr.DataArray(data=[1, 5, 8], coords={"x": [0, 1, 3]}, dims="x")

In [3]: assert_allclose(da1, da2)

AssertionError Traceback (most recent call last) <ipython-input-6-55cbcc1eeb58> in <module> ----> 1 assert_allclose(da1, da2)

~/Documents/Programming/xarray/xarray/testing.py in assert_allclose(a, b, rtol, atol, decode_bytes) 123 assert allclose, f"{a.values}\n{b.values}" 124 elif isinstance(a, DataArray): --> 125 assert_allclose(a.variable, b.variable, **kwargs) 126 assert set(a.coords) == set(b.coords) 127 for v in a.coords.variables:

~/Documents/Programming/xarray/xarray/testing.py in assert_allclose(a, b, rtol, atol, decode_bytes) 121 assert a.dims == b.dims 122 allclose = _data_allclose_or_equiv(a.values, b.values, kwargs) --> 123 assert allclose, f"{a.values}\n{b.values}" 124 elif isinstance(a, DataArray): 125 assert_allclose(a.variable, b.variable, kwargs)

AssertionError: [2 5 8] [1 5 8]

In [4]: assert_identical(da1, da2)

AssertionError Traceback (most recent call last) <ipython-input-7-127e2b1ca0dc> in <module> ----> 1 assert_identical(da1, da2)

~/Documents/Programming/xarray/xarray/testing.py in assert_identical(a, b) 83 elif isinstance(a, DataArray): 84 assert a.name == b.name ---> 85 assert a.identical(b), formatting.diff_array_repr(a, b, "identical") 86 elif isinstance(a, (Dataset, Variable)): 87 assert a.identical(b), formatting.diff_dataset_repr(a, b, "identical")

AssertionError: Left and right DataArray objects are not identical

Differing values: L array([2, 5, 8]) R array([1, 5, 8]) Differing coordinates: L * x (x) int64 0 1 2 R * x (x) int64 0 1 3 `` It would be good to somehow extendformatting.diff_array_reprandformatting.diff_dataset_reprand to then use those to print a diff forassert_allclose`.

I found a reference to this: https://github.com/pydata/xarray/pull/1690#issuecomment-455540813, but it's not being actively worked on.

At the moment, the documentation for methods introduced by accessors is broken. We work around this issue by directly referencing the corresponding accessor method or the underlying function. Of course, this is not how we recommend calling them, so we should try to fix this.

It seems this is mostly a sphinx issue since this is known (see #3333 and #3602), but we didn't find a way to resolve this. I'd like use this issue to collect ideas for solutions.

It seems that after the merge of #3818, the RTD builds started to time out while the docs CI take about 37 minutes.

As a reference, before #3818 our docs CI completed in about 6 minutes.

I'm not sure if that is due to #3818 or because of updated dependencies (dask?), but I think we should try not to take these 30 minutes.

So after #4124 has been fixed upstream, we now have problems with installing numpy / pandas:

I'm not sure if I read the logs correctly, but it seems pandas (or the process to build their wheel, see their pyproject.toml) depends on numpy==1.15.4 which doesn't have a wheel for py38 and thus the source distribution is chosen. I'm not sure why the building the wheel from sdist doesn't work anymore, though.

For reference, see the logs of the last passing build.

I'm not sure why, but our upstream-dev CI fails when importing dask.array: https://dev.azure.com/xarray/xarray/_build/results?buildId=2996&view=logs&j=2280efed-fda1-53bd-9213-1fa8ec9b4fa8&t=aa9ddc6e-3e6c-56cb-4312-111c01d6f735

We have several branches on our main repository that appear to be either out-dated or used to build documentation on RTD (these are no longer necessary since RTD fixed the conda-out-of-memory issues for everyone: building on personal RTD setups should work). Since they already lead to confusion, I think we should try clean them up. Here's a list of the branches I think we can remove: - [x] 0.11.x: seems to be for the 0.11 releases which are no longer maintained (points to the same commit as the v0.11.3 tag) - [ ] accessor-documentation: I intended to use this for building the documentation on RTD, but I accidentally opened #3988 from it instead of from the branch on my own repository. It will be removed once that PR is merged / closed. - [x] fix-docs is from more than a year ago. @shoyer, is there anything that was not already merged into master? - [x] fix-rtd was a attempt to fix RTD when it continuously failed about 1-2 months ago - [x] revert-3128-drop-keyword-support: I think that was used to give a contributor the possibility to amend to a already merged PR, but in the end a follow-up PR was used instead. - [x] scipy19-docs and scipy19-docs-backup: these were merged last year (but turns out there are a few pending PRs that try to merge into scipy19-docs)

Edit: I'm going to delete those branches on Monday

Since dask/distributed#3706 gen_cluster requires functions decorated with it to be coroutines. We use it twice in test_distributed.py: on test_async and on test_serializable_locks, which both are pre-3.5 style coroutines (yield / yield from instead of async / await) but not marked as such (I'm not sure what pytest does with coroutines).

I think this should be a pretty strait-forward fix: replace def with async def and yield / yield from with await.

We recently got a few reports about tests failing with most optional dependencies installed but not dask (#3777, #3778, #3779).

Since dask is pretty tightly coupled in our code, I think it might be worth to add a CI job that runs the tests in a environment with all optional dependencies except from dask. Thoughts?

Another set of failing cftime tests, this time because of warnings while calling open_dataset (I think?): FAILED xarray/tests/test_backends.py::test_use_cftime_standard_calendar_default_in_range[gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_standard_calendar_default_in_range[proleptic_gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_standard_calendar_default_in_range[standard] FAILED xarray/tests/test_backends.py::test_use_cftime_true[1500-360_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[1500-standard] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-360_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-365_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-366_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-all_leap] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-julian] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-noleap] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-proleptic_gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2000-standard] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-360_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-365_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-366_day] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-all_leap] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-julian] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-noleap] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-proleptic_gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_true[2500-standard] FAILED xarray/tests/test_backends.py::test_use_cftime_false_standard_calendar_in_range[gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_false_standard_calendar_in_range[proleptic_gregorian] FAILED xarray/tests/test_backends.py::test_use_cftime_false_standard_calendar_in_range[standard]

```pytb __ test_use_cftime_standard_calendar_default_in_range[gregorian] ___

calendar = 'gregorian' for v in ["x", "time"]: original[v].attrs["units"] = units original[v].attrs["calendar"] = calendar

    x_timedeltas = np.array(x).astype("timedelta64[D]")
    time_timedeltas = np.array(time).astype("timedelta64[D]")
    decoded_x = np.datetime64(units_date, "ns") + x_timedeltas
    decoded_time = np.datetime64(units_date, "ns") + time_timedeltas
    expected_x = DataArray(decoded_x, [("time", decoded_time)], name="x")
    expected_time = DataArray(decoded_time, [("time", decoded_time)], name="time")

    with create_tmp_file() as tmp_file:
        original.to_netcdf(tmp_file)
        with pytest.warns(None) as record:
            with open_dataset(tmp_file, use_cftime=False) as ds:
                assert_identical(expected_x, ds.x)
                assert_identical(expected_time, ds.time)

          assert not record

E assert not WarningsChecker(record=True)

xarray/tests/test_backends.py:4378: AssertionError

```

The py37-windows CI seems to fail for newer PRs: ```pytb ___ TestPseudoNetCDFFormat.test_uamiv_format_write ____

self = <xarray.tests.test_backends.TestPseudoNetCDFFormat object at 0x000002E11FF2DC08>

def test_uamiv_format_write(self):
fmtkw = {"format": "uamiv"}

expected = open_example_dataset(
        "example.uamiv", engine="pseudonetcdf", backend_kwargs=fmtkw
)
with self.roundtrip(
    expected,
        save_kwargs=fmtkw,
    open_kwargs={"backend_kwargs": fmtkw},
        allow_cleanup_failure=True,
) as actual:

      assert_identical(expected, actual)

xarray\tests\test_backends.py:3532:

xarray\core\formatting.py:628: in diff_dataset_repr summary.append(diff_attrs_repr(a.attrs, b.attrs, compat))

a_mapping = {'CPROJ': 0, 'FILEDESC': 'CAMx ', 'FTYPE': 1, 'GDNAM': 'CAMx ', ...} b_mapping = {'CPROJ': 0, 'FILEDESC': 'CAMx ', 'FTYPE': 1, 'GDNAM': 'CAMx ', ...} compat = 'identical', title = 'Attributes' summarizer = <function summarize_attr at 0x000002E1156813A8>, col_width = None

def _diff_mapping_repr(a_mapping, b_mapping, compat, title, summarizer, col_width=None):
    def extra_items_repr(extra_keys, mapping, ab_side):
        extra_repr = [summarizer(k, mapping[k], col_width) for k in extra_keys]
        if extra_repr:
            header = f"{title} only on the {ab_side} object:"
            return [header] + extra_repr
        else:
            return []

a_keys = set(a_mapping)
    b_keys = set(b_mapping)

    summary = []

diff_items = []

    for k in a_keys & b_keys:
    try:
            # compare xarray variable
            compatible = getattr(a_mapping[k], compat)(b_mapping[k])
            is_variable = True
        except AttributeError:
            # compare attribute value
            compatible = a_mapping[k] == b_mapping[k]
            is_variable = False

      if not compatible:

E ValueError: The truth value of an array with more than one element is ambiguous. Use a.any() or a.all() ```

Running the MacOSX py38 CI job gives us this warning: ```

[warning]This pipeline uses a Microsoft-hosted agent image that will be removed on March 23, 2020 (MacOS-10.13). You must make changes to your pipeline before that date, or else your pipeline will fail. Learn more (https://aka.ms/removing-older-images-hosted-pools).

```

RTD released a new version of the configuration file format a bit more than a year ago and deprecated the old version. I think we should try to migrate our setup to the new format. Thoughts?

At the moment, the formatting.diff_*_repr functions that provide the pretty-printing for assert_* use repr to format NEP-18 strings, truncating the result if it is too long. In the case of pint's quantities, this makes the pretty printing useless since only a few values are visible and the unit is in the truncated part.

What should we about this? Does pint have to change its repr?

xref #3611

In #3199 the documentation on readthedocs did not build because the newest packaged version of xarray was pulled in by a dependency -- in this case cfgrib. I'm not sure why, but this shadows the versions installed by at least python setup.py install --force and python -m pip install -e . (not python -m pip install ., I think, but readthedocs uses --update-strategy=eager for pip-installing, which deactivates version pinning).

Fortunately, cfgrib does not have a hard dependency on xarray (it's in the extras_require section), so this issue was bypassed in #3557 by using pip to install it. Should we ever want to introduce a dependency that requires xarray (or to use conda to install cfgrib), this is bound to resurface.

It might be that this is a problem with how versioneer constructs versions based on git commit hashes and a fix to #2853 is a fix to this, but it certainly needs more investigation.

3369 is related, but more about catching these sort of issues before merging.

The docs claim quantile works on grouped datasets, but that does not seem to be the case: ```python

import xarray as xr ds = xr.Dataset(data_vars={"a": ("x", list("abcd"))}, coords={"x": range(4)}) ds.a.groupby(ds.x % 2 == 0).quantile <bound method DataArrayGroupBy.quantile of DataArrayGroupBy, grouped over 'x' 2 groups with labels False, True.> ds.groupby(ds.x % 2 == 0).quantile AttributeError: 'DatasetGroupBy' object has no attribute 'quantile' ``` this was found while trying to silence the nit-picky sphinx warnings in #3516

I'm not sure if using assign_coords to overwrite the data of coords is the best way to do so, but using mixed args (on current master) turns out to have surprising results: ```python

obj = xr.DataArray( ... data=[6, 3, 4, 6], ... coords={"x": list("abcd"), "y": ("x", range(4))}, ... dims="x", ... ) obj <xarray.DataArray 'obj' (x: 4)> array([6, 3, 4, 6]) Coordinates: * x (x) <U1 'a' 'b' 'c' 'd' y (x) int64 0 1 2 3

works as expected

obj.assign_coords(coords={"x": list("efgh"), "y": ("x", [0, 2, 4, 6])}) <xarray.DataArray 'obj' (x: 4)> array([6, 3, 4, 6]) Coordinates: * x (x) <U1 'e' 'f' 'g' 'h' y (x) int64 0 2 4 6

works, too (same as .data / .values)

obj.assign_coords(coords={ ... "x": obj.x.copy(data=list("efgh")).variable, ... "y": ("x", [0, 2, 4, 6]), ... }) <xarray.DataArray 'obj' (x: 4)> array([6, 3, 4, 6]) Coordinates: * x (x) <U1 'e' 'f' 'g' 'h' y (x) int64 0 2 4 6

this drops "y"

obj.assign_coords(coords={ ... "x": obj.x.copy(data=list("efgh")), ... "y": ("x", [0, 2, 4, 6]), ... }) <xarray.DataArray 'obj' (x: 4)> array([6, 3, 4, 6]) Coordinates: * x (x) <U1 'e' 'f' 'g' 'h' Passing a `DataArray` for `y`, like `obj.y * 2` while also changing `x` (the type does not matter) always results in a `MergeError`:python obj.assign_coords(x=list("efgh"), y=obj.y * 2) xarray.core.merge.MergeError: conflicting values for index 'x' on objects to be combined: first value: Index(['e', 'f', 'g', 'h'], dtype='object', name='x') second value: Index(['a', 'b', 'c', 'd'], dtype='object', name='x') ```

I would expect the result to be the same regardless of the type of the new coords.

While working on #3238, I tried replacing np.arange with np.linspace to create test arrays: ```python

ureg = pint.UnitRegistry()

with int values

array = np.arange(10).astype(int) * ureg.m np.max(array) <Quantity(9, 'meter')> np.max(xr.DataArray(data=array)) # works as expected <xarray.DataArray ()> <Quantity(9, 'meter')>

now with floats

array = np.arange(10).astype(float) * ureg.m np.max(array) <Quantity(9.0, 'meter')> np.max(xr.DataArray(data=array)) # unit information is lost <xarray.DataArray ()> array(9.) `` Judging by the build logs of #3238, this seems to be the case for all aggregation functions except fromnp.median` and of course those that return booleans or indices.