From e57b0994440dc8e9ecf655ec85b251909b052dd1 Mon Sep 17 00:00:00 2001 From: dcherian Date: Fri, 22 Nov 2019 08:26:25 -0700 Subject: [PATCH 01/16] Add some xfailed tests. --- xarray/tests/test_dask.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 4c1f317342f..bee12c15334 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1375,3 +1375,41 @@ def test_lazy_array_equiv_merge(compat): xr.merge([da1, da3], compat=compat) with raise_if_dask_computes(max_computes=2): xr.merge([da1, da2 / 2], compat=compat) + + +@pytest.mark.xfail +@pytest.mark.parametrize("obj", [make_da(), make_ds()]) +@pytest.mark.parametrize( + "transform", + [ + lambda a: a.copy(), + lambda a: a.isel(x=np.arange(a.x)), + lambda a: a.isel(x=slice(None, None)), + lambda a: a.sel(x=a.x), + lambda a: a.sel(x=a.x.values), + lambda a: a.transpose(...), + lambda a: a.squeeze(), + lambda a: a.sortby("x"), + lambda a: a.reindex(x=a.x), + lambda a: a.reindex_like(a), + lambda a: a.pipe(lambda x: x), + lambda a: a.map(lambda x: x), + lambda a: a.reduce(lambda x: x), + lambda a: a["a"].broadcast_like(a["a"]), + lambda a: a.groupby("x").map(lambda x: x), + lambda a: a.where(xr.full_like(a, fill_value=True)), + lambda a: xr.align([a, a])[0], + lambda a: xr.broadcast([a["a"], a["a"]])[0], + lambda a: xr.where(xr.full_like(a, fill_value=True), a, np.nan), + # assign, assign_coords, assign_attrs, update + # rename, rename_vars, rename_dims, + # swap_dims, expand_dims + # set_coords / reset_coords + # set_index / reset_index + # stack / unstack + # to_temp_datasets, from_temp_dataset + ], +) +def test_transforms_pass_lazy_array_equiv(obj, transform): + with raise_if_dask_computes(): + assert_equal(obj, transform(obj)) From 6ba4509859e71d4808398d0980313ee8fe6ab7da Mon Sep 17 00:00:00 2001 From: dcherian Date: Sat, 23 Nov 2019 15:04:20 -0700 Subject: [PATCH 02/16] Only xfail failing tests. --- xarray/tests/test_dask.py | 44 +++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index bee12c15334..55649e7d23c 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1377,37 +1377,49 @@ def test_lazy_array_equiv_merge(compat): xr.merge([da1, da2 / 2], compat=compat) -@pytest.mark.xfail @pytest.mark.parametrize("obj", [make_da(), make_ds()]) @pytest.mark.parametrize( "transform", [ lambda a: a.copy(), - lambda a: a.isel(x=np.arange(a.x)), + pytest.param(lambda a: a.isel(x=np.arange(a.x)), marks=pytest.mark.xfail), lambda a: a.isel(x=slice(None, None)), - lambda a: a.sel(x=a.x), - lambda a: a.sel(x=a.x.values), + pytest.param(lambda a: a.sel(x=a.x), marks=pytest.mark.xfail), + pytest.param(lambda a: a.sel(x=a.x.values), marks=pytest.mark.xfail), lambda a: a.transpose(...), lambda a: a.squeeze(), - lambda a: a.sortby("x"), + pytest.param(lambda a: a.sortby("x"), marks=pytest.mark.xfail), lambda a: a.reindex(x=a.x), lambda a: a.reindex_like(a), lambda a: a.pipe(lambda x: x), - lambda a: a.map(lambda x: x), - lambda a: a.reduce(lambda x: x), - lambda a: a["a"].broadcast_like(a["a"]), - lambda a: a.groupby("x").map(lambda x: x), - lambda a: a.where(xr.full_like(a, fill_value=True)), - lambda a: xr.align([a, a])[0], - lambda a: xr.broadcast([a["a"], a["a"]])[0], - lambda a: xr.where(xr.full_like(a, fill_value=True), a, np.nan), + pytest.param(lambda a: a["a"].broadcast_like(a["a"]), marks=pytest.mark.xfail), + pytest.param( + lambda a: a.groupby("x").map(lambda x: x), marks=pytest.mark.xfail + ), + pytest.param( + lambda a: a.where(xr.full_like(a, fill_value=True)), marks=pytest.mark.xfail + ), + pytest.param(lambda a: xr.align([a, a])[0], marks=pytest.mark.xfail), + pytest.param( + lambda a: xr.broadcast([a["a"], a["a"]])[0], marks=pytest.mark.xfail + ), + pytest.param( + lambda a: xr.where(xr.full_like(a, fill_value=True), a, np.nan), + marks=pytest.mark.xfail, + ), + pytest.param( + lambda a: a.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), + marks=pytest.mark.xfail, + ), # assign, assign_coords, assign_attrs, update - # rename, rename_vars, rename_dims, + # rename, # swap_dims, expand_dims - # set_coords / reset_coords # set_index / reset_index # stack / unstack - # to_temp_datasets, from_temp_dataset + # to_temp_dataset, from_temp_dataset + # Dataset lambda a: a.map(lambda x: x) + # Dataset lambda a: a.rename_vars({"a": "a1"}).rename_vars({"a1": "a"}), + # Dataset lambda a: a.set_coords("a").reset_coords("a"), ], ) def test_transforms_pass_lazy_array_equiv(obj, transform): From 61b73347d71612e87d1478c0721b5c86ec6ee4bc Mon Sep 17 00:00:00 2001 From: dcherian Date: Mon, 25 Nov 2019 08:53:14 -0700 Subject: [PATCH 03/16] Add DataArray.rename_dims, DataArray.rename_vars --- xarray/core/dataarray.py | 59 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 1b135a350d1..190a3e78be3 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1466,6 +1466,65 @@ def rename( new_name_or_name_dict = cast(Hashable, new_name_or_name_dict) return self._replace(name=new_name_or_name_dict) + def rename_dims( + self, dims_dict: Mapping[Hashable, Hashable] = None, **dims: Hashable + ) -> "DataArray": + """Returns a new object with renamed dimensions only. + + Parameters + ---------- + dims_dict : dict-like, optional + Dictionary whose keys are current dimension names and + whose values are the desired names. + **dims, optional + Keyword form of ``dims_dict``. + One of dims_dict or dims must be provided. + + Returns + ------- + renamed : Dataset + Dataset with renamed dimensions. + + See Also + -------- + Dataset.swap_dims + Dataset.rename + Dataset.rename_vars + DataArray.rename + """ + dims_dict = either_dict_or_kwargs(dims_dict, dims, "rename_dims") + ds = self._to_temp_dataset().rename_dims(dims_dict) + return self._from_temp_dataset(ds) + + def rename_vars( + self, name_dict: Mapping[Hashable, Hashable] = None, **names: Hashable + ) -> "DataArray": + """Returns a new object with renamed variables including coordinates + + Parameters + ---------- + name_dict : dict-like, optional + Dictionary whose keys are current variable or coordinate names and + whose values are the desired names. + **names, optional + Keyword form of ``name_dict``. + One of name_dict or names must be provided. + + Returns + ------- + renamed : Dataset + Dataset with renamed variables including coordinates + + See Also + -------- + Dataset.swap_dims + Dataset.rename + Dataset.rename_dims + DataArray.rename + """ + ds = self._to_temp_dataset().rename_vars(name_dict) + return self._from_temp_dataset(ds) + def swap_dims(self, dims_dict: Mapping[Hashable, Hashable]) -> "DataArray": """Returns a new DataArray with swapped dimensions. From 6f3641613112a71bdbeee95285091c9d1b9034b4 Mon Sep 17 00:00:00 2001 From: dcherian Date: Mon, 25 Nov 2019 08:53:29 -0700 Subject: [PATCH 04/16] Update tests. --- xarray/tests/test_dask.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 55649e7d23c..40b3e3d042e 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1407,10 +1407,10 @@ def test_lazy_array_equiv_merge(compat): lambda a: xr.where(xr.full_like(a, fill_value=True), a, np.nan), marks=pytest.mark.xfail, ), - pytest.param( - lambda a: a.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), - marks=pytest.mark.xfail, - ), + lambda a: a.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), + lambda a: a.rename_vars({"cxy": "cnew"}).rename_vars( + {"cnew": "cxy"} + ), # indexes fail # assign, assign_coords, assign_attrs, update # rename, # swap_dims, expand_dims From 165fbbafafccb31aeac70c62ccfdc59e273b5bab Mon Sep 17 00:00:00 2001 From: dcherian Date: Mon, 25 Nov 2019 20:26:51 -0700 Subject: [PATCH 05/16] todos --- xarray/tests/test_dask.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 40b3e3d042e..dd0031231c0 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1399,7 +1399,8 @@ def test_lazy_array_equiv_merge(compat): pytest.param( lambda a: a.where(xr.full_like(a, fill_value=True)), marks=pytest.mark.xfail ), - pytest.param(lambda a: xr.align([a, a])[0], marks=pytest.mark.xfail), + lambda a: xr.align(a, a)[0], # remove? + lambda a: xr.align(a, xr.zeros_like(a))[0], pytest.param( lambda a: xr.broadcast([a["a"], a["a"]])[0], marks=pytest.mark.xfail ), @@ -1425,3 +1426,8 @@ def test_lazy_array_equiv_merge(compat): def test_transforms_pass_lazy_array_equiv(obj, transform): with raise_if_dask_computes(): assert_equal(obj, transform(obj)) + + +# TODO: transpose_coords for dataset +# getitem with mask +# __getitem__ From 15c4018fb200d7164e853dc9096db426150ae1ab Mon Sep 17 00:00:00 2001 From: dcherian Date: Mon, 25 Nov 2019 20:11:56 -0700 Subject: [PATCH 06/16] Fix isel. Tests pass. --- xarray/core/variable.py | 17 +++++++++++++++-- xarray/tests/test_dask.py | 8 ++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 55e8f64d56c..41cc0ae6879 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -5,7 +5,7 @@ from collections import defaultdict from datetime import timedelta from distutils.version import LooseVersion -from typing import Any, Dict, Hashable, Mapping, TypeVar, Union +from typing import Any, Dict, Hashable, Iterable, Mapping, TypeVar, Union import numpy as np import pandas as pd @@ -540,6 +540,17 @@ def _broadcast_indexes(self, key): k.item() if isinstance(k, np.ndarray) and k.ndim == 0 else k for k in key ) + key_dict = dict(zip(self.dims, key)) + for dim, k in key_dict.items(): + if isinstance(k, Iterable) or ( + isinstance(k, Variable) and k.dims == (dim,) # catch da.sel(x=da.x) + ): + if duck_array_ops.array_equiv(k, np.arange(self.sizes[dim])): + # short-circuit when keys are effectively slice(None) + # This preserves dask name and allows lazy array equivalence checks + key_dict[dim] = slice(None) + key = tuple(key_dict.values()) + if all(isinstance(k, BASIC_INDEXING_TYPES) for k in key): return self._broadcast_indexes_basic(key) @@ -1048,7 +1059,9 @@ def isel( invalid = indexers.keys() - set(self.dims) if invalid: - raise ValueError("dimensions %r do not exist" % invalid) + raise ValueError( + f"dimensions {invalid} do not exist. Expected one or more of {self.dims} " + ) key = tuple(indexers.get(dim, slice(None)) for dim in self.dims) return self[key] diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index dd0031231c0..cee3b7c3c04 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1382,10 +1382,10 @@ def test_lazy_array_equiv_merge(compat): "transform", [ lambda a: a.copy(), - pytest.param(lambda a: a.isel(x=np.arange(a.x)), marks=pytest.mark.xfail), - lambda a: a.isel(x=slice(None, None)), - pytest.param(lambda a: a.sel(x=a.x), marks=pytest.mark.xfail), - pytest.param(lambda a: a.sel(x=a.x.values), marks=pytest.mark.xfail), + lambda a: a.isel(x=np.arange(a.sizes["x"])), + lambda a: a.isel(x=slice(None)), + lambda a: a.sel(x=a.x), + lambda a: a.sel(x=a.x.values), lambda a: a.transpose(...), lambda a: a.squeeze(), pytest.param(lambda a: a.sortby("x"), marks=pytest.mark.xfail), From 3754662df08fd40dd1a989619c58a701f45aeec1 Mon Sep 17 00:00:00 2001 From: dcherian Date: Sat, 30 Nov 2019 14:15:30 -0500 Subject: [PATCH 07/16] All tests pass. --- xarray/core/variable.py | 8 +++++--- xarray/tests/test_dask.py | 36 ++++++++++++++---------------------- 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 41cc0ae6879..1a9aa917601 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -542,9 +542,11 @@ def _broadcast_indexes(self, key): key_dict = dict(zip(self.dims, key)) for dim, k in key_dict.items(): - if isinstance(k, Iterable) or ( - isinstance(k, Variable) and k.dims == (dim,) # catch da.sel(x=da.x) - ): + if isinstance(k, Iterable): + # let da.sel(x=da.x) pass but skip if Variable has different dimensions + # e.g. da.sel(x=Variable(("points",), [0, 1, 2])) + if isinstance(k, Variable) and k.dims != (dim,): + continue if duck_array_ops.array_equiv(k, np.arange(self.sizes[dim])): # short-circuit when keys are effectively slice(None) # This preserves dask name and allows lazy array equivalence checks diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index cee3b7c3c04..96a2c3c5380 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1388,39 +1388,21 @@ def test_lazy_array_equiv_merge(compat): lambda a: a.sel(x=a.x.values), lambda a: a.transpose(...), lambda a: a.squeeze(), - pytest.param(lambda a: a.sortby("x"), marks=pytest.mark.xfail), + lambda a: a.sortby("x"), lambda a: a.reindex(x=a.x), lambda a: a.reindex_like(a), lambda a: a.pipe(lambda x: x), - pytest.param(lambda a: a["a"].broadcast_like(a["a"]), marks=pytest.mark.xfail), - pytest.param( - lambda a: a.groupby("x").map(lambda x: x), marks=pytest.mark.xfail - ), - pytest.param( - lambda a: a.where(xr.full_like(a, fill_value=True)), marks=pytest.mark.xfail - ), - lambda a: xr.align(a, a)[0], # remove? lambda a: xr.align(a, xr.zeros_like(a))[0], pytest.param( - lambda a: xr.broadcast([a["a"], a["a"]])[0], marks=pytest.mark.xfail - ), - pytest.param( - lambda a: xr.where(xr.full_like(a, fill_value=True), a, np.nan), - marks=pytest.mark.xfail, + lambda a: a.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), + marks=pytest.mark.xfail, # x index is lost. ), - lambda a: a.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), - lambda a: a.rename_vars({"cxy": "cnew"}).rename_vars( - {"cnew": "cxy"} - ), # indexes fail + lambda a: a.rename_vars({"cxy": "cnew"}).rename_vars({"cnew": "cxy"}), # assign, assign_coords, assign_attrs, update # rename, # swap_dims, expand_dims # set_index / reset_index # stack / unstack - # to_temp_dataset, from_temp_dataset - # Dataset lambda a: a.map(lambda x: x) - # Dataset lambda a: a.rename_vars({"a": "a1"}).rename_vars({"a1": "a"}), - # Dataset lambda a: a.set_coords("a").reset_coords("a"), ], ) def test_transforms_pass_lazy_array_equiv(obj, transform): @@ -1428,6 +1410,16 @@ def test_transforms_pass_lazy_array_equiv(obj, transform): assert_equal(obj, transform(obj)) +def test_more_transforms_pass_lazy_array_equiv(map_da, map_ds): + with raise_if_dask_computes(): + assert_equal(map_ds.cxy, map_ds.cxy.broadcast_like(map_ds.cxy)) + assert_equal(map_ds.a, xr.broadcast(map_ds.a, map_ds.a)[0]) + assert_equal(map_ds.map(lambda x: x), map_ds) + assert_equal(map_ds.set_coords("a").reset_coords("a"), map_ds) + + assert_equal(map_da._from_temp_dataset(map_da._to_temp_dataset()), map_da) + + # TODO: transpose_coords for dataset # getitem with mask # __getitem__ From ae946201019e53752f6e32f0b3e884257f3804da Mon Sep 17 00:00:00 2001 From: dcherian Date: Sat, 30 Nov 2019 14:15:45 -0500 Subject: [PATCH 08/16] Add comments. --- xarray/tests/test_dask.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 96a2c3c5380..e6ccbc6f266 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1387,8 +1387,8 @@ def test_lazy_array_equiv_merge(compat): lambda a: a.sel(x=a.x), lambda a: a.sel(x=a.x.values), lambda a: a.transpose(...), - lambda a: a.squeeze(), - lambda a: a.sortby("x"), + lambda a: a.squeeze(), # no dimensions to squeeze + lambda a: a.sortby("x"), # "x" is already sorted lambda a: a.reindex(x=a.x), lambda a: a.reindex_like(a), lambda a: a.pipe(lambda x: x), From cbfbc50da6f5e8efbebe6fbe981b6e3bca0893b3 Mon Sep 17 00:00:00 2001 From: dcherian Date: Sat, 30 Nov 2019 14:19:55 -0500 Subject: [PATCH 09/16] wip --- xarray/tests/test_dask.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index e6ccbc6f266..b3a1e2800f7 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1402,7 +1402,6 @@ def test_lazy_array_equiv_merge(compat): # rename, # swap_dims, expand_dims # set_index / reset_index - # stack / unstack ], ) def test_transforms_pass_lazy_array_equiv(obj, transform): @@ -1413,11 +1412,16 @@ def test_transforms_pass_lazy_array_equiv(obj, transform): def test_more_transforms_pass_lazy_array_equiv(map_da, map_ds): with raise_if_dask_computes(): assert_equal(map_ds.cxy, map_ds.cxy.broadcast_like(map_ds.cxy)) + # assert_equal( + # map_ds.transpose(reversed(list(map_ds.dims)), transpose_coords=False).cxy, + # map_ds.cxy, + # ) assert_equal(map_ds.a, xr.broadcast(map_ds.a, map_ds.a)[0]) assert_equal(map_ds.map(lambda x: x), map_ds) assert_equal(map_ds.set_coords("a").reset_coords("a"), map_ds) assert_equal(map_da._from_temp_dataset(map_da._to_temp_dataset()), map_da) + assert_equal(map_da.astype(map_da.dtype), map_da) # TODO: transpose_coords for dataset From 23fbd07ad951eabc3b22f08597d70a3be1261f29 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 3 Dec 2019 09:58:17 -0500 Subject: [PATCH 10/16] cleanup --- xarray/tests/test_dask.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index b3a1e2800f7..3227589e71a 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1377,6 +1377,7 @@ def test_lazy_array_equiv_merge(compat): xr.merge([da1, da2 / 2], compat=compat) +@pytest.mark.filterwarnings("ignore::FutureWarning") @pytest.mark.parametrize("obj", [make_da(), make_ds()]) @pytest.mark.parametrize( "transform", @@ -1393,11 +1394,6 @@ def test_lazy_array_equiv_merge(compat): lambda a: a.reindex_like(a), lambda a: a.pipe(lambda x: x), lambda a: xr.align(a, xr.zeros_like(a))[0], - pytest.param( - lambda a: a.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), - marks=pytest.mark.xfail, # x index is lost. - ), - lambda a: a.rename_vars({"cxy": "cnew"}).rename_vars({"cnew": "cxy"}), # assign, assign_coords, assign_attrs, update # rename, # swap_dims, expand_dims @@ -1412,16 +1408,20 @@ def test_transforms_pass_lazy_array_equiv(obj, transform): def test_more_transforms_pass_lazy_array_equiv(map_da, map_ds): with raise_if_dask_computes(): assert_equal(map_ds.cxy, map_ds.cxy.broadcast_like(map_ds.cxy)) - # assert_equal( - # map_ds.transpose(reversed(list(map_ds.dims)), transpose_coords=False).cxy, - # map_ds.cxy, - # ) assert_equal(map_ds.a, xr.broadcast(map_ds.a, map_ds.a)[0]) assert_equal(map_ds.map(lambda x: x), map_ds) assert_equal(map_ds.set_coords("a").reset_coords("a"), map_ds) + # fails because of index error + # assert_equal(map_ds.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), + # map_ds) + assert_equal( + map_ds.rename_vars({"cxy": "cnew"}).rename_vars({"cnew": "cxy"}), map_ds + ) + assert_equal(map_da._from_temp_dataset(map_da._to_temp_dataset()), map_da) assert_equal(map_da.astype(map_da.dtype), map_da) + assert_equal(map_da.transpose("y", "x", transpose_coords=False).cxy, map_da.cxy) # TODO: transpose_coords for dataset From 5ba42ea7fca4522adcd61f77ec8d65455f1cc927 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 3 Dec 2019 09:58:50 -0500 Subject: [PATCH 11/16] Revert "Add DataArray.rename_dims, DataArray.rename_vars" This reverts commit 61b73347d71612e87d1478c0721b5c86ec6ee4bc. --- xarray/core/dataarray.py | 59 ---------------------------------------- 1 file changed, 59 deletions(-) diff --git a/xarray/core/dataarray.py b/xarray/core/dataarray.py index 190a3e78be3..1b135a350d1 100644 --- a/xarray/core/dataarray.py +++ b/xarray/core/dataarray.py @@ -1466,65 +1466,6 @@ def rename( new_name_or_name_dict = cast(Hashable, new_name_or_name_dict) return self._replace(name=new_name_or_name_dict) - def rename_dims( - self, dims_dict: Mapping[Hashable, Hashable] = None, **dims: Hashable - ) -> "DataArray": - """Returns a new object with renamed dimensions only. - - Parameters - ---------- - dims_dict : dict-like, optional - Dictionary whose keys are current dimension names and - whose values are the desired names. - **dims, optional - Keyword form of ``dims_dict``. - One of dims_dict or dims must be provided. - - Returns - ------- - renamed : Dataset - Dataset with renamed dimensions. - - See Also - -------- - Dataset.swap_dims - Dataset.rename - Dataset.rename_vars - DataArray.rename - """ - dims_dict = either_dict_or_kwargs(dims_dict, dims, "rename_dims") - ds = self._to_temp_dataset().rename_dims(dims_dict) - return self._from_temp_dataset(ds) - - def rename_vars( - self, name_dict: Mapping[Hashable, Hashable] = None, **names: Hashable - ) -> "DataArray": - """Returns a new object with renamed variables including coordinates - - Parameters - ---------- - name_dict : dict-like, optional - Dictionary whose keys are current variable or coordinate names and - whose values are the desired names. - **names, optional - Keyword form of ``name_dict``. - One of name_dict or names must be provided. - - Returns - ------- - renamed : Dataset - Dataset with renamed variables including coordinates - - See Also - -------- - Dataset.swap_dims - Dataset.rename - Dataset.rename_dims - DataArray.rename - """ - ds = self._to_temp_dataset().rename_vars(name_dict) - return self._from_temp_dataset(ds) - def swap_dims(self, dims_dict: Mapping[Hashable, Hashable]) -> "DataArray": """Returns a new DataArray with swapped dimensions. From eb795955c1f7f7a96efd1d19fd3b8cc2732c8437 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 3 Dec 2019 10:06:07 -0500 Subject: [PATCH 12/16] more tests --- xarray/tests/test_dask.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 3227589e71a..43a602eb9b0 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1382,9 +1382,14 @@ def test_lazy_array_equiv_merge(compat): @pytest.mark.parametrize( "transform", [ + lambda a: a.assign_attrs(new_attr="anew"), + lambda a: a.assign_coords(cxy=a.cxy), lambda a: a.copy(), lambda a: a.isel(x=np.arange(a.sizes["x"])), lambda a: a.isel(x=slice(None)), + lambda a: a.loc[dict(x=slice(None))], + lambda a: a.loc[dict(x=np.arange(a.sizes["x"]))], + lambda a: a.loc[dict(x=a.x)], lambda a: a.sel(x=a.x), lambda a: a.sel(x=a.x.values), lambda a: a.transpose(...), @@ -1392,11 +1397,11 @@ def test_lazy_array_equiv_merge(compat): lambda a: a.sortby("x"), # "x" is already sorted lambda a: a.reindex(x=a.x), lambda a: a.reindex_like(a), + lambda a: a.rename({"cxy": "cnew"}).rename({"cnew": "cxy"}), lambda a: a.pipe(lambda x: x), lambda a: xr.align(a, xr.zeros_like(a))[0], - # assign, assign_coords, assign_attrs, update - # rename, - # swap_dims, expand_dims + # assign + # swap_dims # set_index / reset_index ], ) @@ -1407,10 +1412,11 @@ def test_transforms_pass_lazy_array_equiv(obj, transform): def test_more_transforms_pass_lazy_array_equiv(map_da, map_ds): with raise_if_dask_computes(): - assert_equal(map_ds.cxy, map_ds.cxy.broadcast_like(map_ds.cxy)) - assert_equal(map_ds.a, xr.broadcast(map_ds.a, map_ds.a)[0]) + assert_equal(map_ds.cxy.broadcast_like(map_ds.cxy), map_ds.cxy) + assert_equal(xr.broadcast(map_ds.cxy, map_ds.cxy)[0], map_ds.cxy) assert_equal(map_ds.map(lambda x: x), map_ds) assert_equal(map_ds.set_coords("a").reset_coords("a"), map_ds) + assert_equal(map_ds.update({"a": map_ds.a}), map_ds) # fails because of index error # assert_equal(map_ds.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), From 28074b979258cbc120a1b8f8d4a0311e7f25bd6a Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 3 Dec 2019 10:10:36 -0500 Subject: [PATCH 13/16] Add comment --- xarray/core/variable.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 1a9aa917601..1c3ab2ec229 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -549,7 +549,8 @@ def _broadcast_indexes(self, key): continue if duck_array_ops.array_equiv(k, np.arange(self.sizes[dim])): # short-circuit when keys are effectively slice(None) - # This preserves dask name and allows lazy array equivalence checks + # This preserves dask name and passes lazy array equivalence checks + # (see duck_array_ops.lazy_array_equiv) key_dict[dim] = slice(None) key = tuple(key_dict.values()) From 1972fde8ae5bb997bb8560a8b65d0187cfb7f1e0 Mon Sep 17 00:00:00 2001 From: dcherian Date: Wed, 11 Dec 2019 08:03:07 -0700 Subject: [PATCH 14/16] Add optimization to DaskIndexingAdapter --- xarray/core/indexing.py | 20 +++++++++++++++++++- xarray/core/variable.py | 16 +--------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/xarray/core/indexing.py b/xarray/core/indexing.py index f48c9e72af1..d587ee3a02b 100644 --- a/xarray/core/indexing.py +++ b/xarray/core/indexing.py @@ -4,7 +4,7 @@ from collections import defaultdict from contextlib import suppress from datetime import timedelta -from typing import Any, Callable, Sequence, Tuple, Union +from typing import Any, Callable, Iterable, Sequence, Tuple, Union import numpy as np import pandas as pd @@ -1291,6 +1291,24 @@ def __init__(self, array): self.array = array def __getitem__(self, key): + + if not isinstance(key, VectorizedIndexer): + # if possible, short-circuit when keys are effectively slice(None) + # This preserves dask name and passes lazy array equivalence checks + # (see duck_array_ops.lazy_array_equiv) + rewritten_indexer = False + new_indexer = [] + for idim, k in enumerate(key.tuple): + if isinstance(k, Iterable) and duck_array_ops.array_equiv( + k, np.arange(self.array.shape[idim]) + ): + new_indexer.append(slice(None)) + rewritten_indexer = True + else: + new_indexer.append(k) + if rewritten_indexer: + key = type(key)(tuple(new_indexer)) + if isinstance(key, BasicIndexer): return self.array[key.tuple] elif isinstance(key, VectorizedIndexer): diff --git a/xarray/core/variable.py b/xarray/core/variable.py index 1c3ab2ec229..8e62341d5ee 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -5,7 +5,7 @@ from collections import defaultdict from datetime import timedelta from distutils.version import LooseVersion -from typing import Any, Dict, Hashable, Iterable, Mapping, TypeVar, Union +from typing import Any, Dict, Hashable, Mapping, TypeVar, Union import numpy as np import pandas as pd @@ -540,20 +540,6 @@ def _broadcast_indexes(self, key): k.item() if isinstance(k, np.ndarray) and k.ndim == 0 else k for k in key ) - key_dict = dict(zip(self.dims, key)) - for dim, k in key_dict.items(): - if isinstance(k, Iterable): - # let da.sel(x=da.x) pass but skip if Variable has different dimensions - # e.g. da.sel(x=Variable(("points",), [0, 1, 2])) - if isinstance(k, Variable) and k.dims != (dim,): - continue - if duck_array_ops.array_equiv(k, np.arange(self.sizes[dim])): - # short-circuit when keys are effectively slice(None) - # This preserves dask name and passes lazy array equivalence checks - # (see duck_array_ops.lazy_array_equiv) - key_dict[dim] = slice(None) - key = tuple(key_dict.values()) - if all(isinstance(k, BASIC_INDEXING_TYPES) for k in key): return self._broadcast_indexes_basic(key) From 96d8ded86d846c88bbb8324eb40d78aea39f65d7 Mon Sep 17 00:00:00 2001 From: Deepak Cherian Date: Fri, 17 Jan 2020 09:22:13 -0700 Subject: [PATCH 15/16] Update xarray/core/variable.py Co-Authored-By: crusaderky --- xarray/core/variable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xarray/core/variable.py b/xarray/core/variable.py index ec8829c3dfb..b3d08a474ce 100644 --- a/xarray/core/variable.py +++ b/xarray/core/variable.py @@ -1055,7 +1055,7 @@ def isel( invalid = indexers.keys() - set(self.dims) if invalid: raise ValueError( - f"dimensions {invalid} do not exist. Expected one or more of {self.dims} " + f"dimensions {invalid} do not exist. Expected one or more of {self.dims}" ) key = tuple(indexers.get(dim, slice(None)) for dim in self.dims) From 2ce560ffcff6d14f625034f09d0e716a706e8a38 Mon Sep 17 00:00:00 2001 From: dcherian Date: Tue, 21 Jan 2020 08:00:52 -0700 Subject: [PATCH 16/16] minor. --- xarray/tests/test_dask.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/xarray/tests/test_dask.py b/xarray/tests/test_dask.py index 4daa786068f..33fb88b634d 100644 --- a/xarray/tests/test_dask.py +++ b/xarray/tests/test_dask.py @@ -1392,7 +1392,7 @@ def test_lazy_array_equiv_merge(compat): xr.merge([da1, da2 / 2], compat=compat) -@pytest.mark.filterwarnings("ignore::FutureWarning") +@pytest.mark.filterwarnings("ignore::FutureWarning") # transpose_coords @pytest.mark.parametrize("obj", [make_da(), make_ds()]) @pytest.mark.parametrize( "transform", @@ -1434,8 +1434,10 @@ def test_more_transforms_pass_lazy_array_equiv(map_da, map_ds): assert_equal(map_ds.update({"a": map_ds.a}), map_ds) # fails because of index error - # assert_equal(map_ds.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), - # map_ds) + # assert_equal( + # map_ds.rename_dims({"x": "xnew"}).rename_dims({"xnew": "x"}), map_ds + # ) + assert_equal( map_ds.rename_vars({"cxy": "cnew"}).rename_vars({"cnew": "cxy"}), map_ds ) @@ -1443,8 +1445,3 @@ def test_more_transforms_pass_lazy_array_equiv(map_da, map_ds): assert_equal(map_da._from_temp_dataset(map_da._to_temp_dataset()), map_da) assert_equal(map_da.astype(map_da.dtype), map_da) assert_equal(map_da.transpose("y", "x", transpose_coords=False).cxy, map_da.cxy) - - -# TODO: transpose_coords for dataset -# getitem with mask -# __getitem__