From b5b466ee2ccafcb16c39cc93a1f739477704b1ea Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 27 May 2020 08:59:36 +0200 Subject: [PATCH 1/5] Revert "CLN: _consolidate_inplace less (#34389)" This reverts commit 760ba37a776947694ec816eaf29ce8937e08f544. --- pandas/core/generic.py | 9 +++++++++ pandas/core/internals/managers.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 79805bec85af0..8aa8f8bb60654 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3369,6 +3369,8 @@ class max_speed nv.validate_take(tuple(), kwargs) + self._consolidate_inplace() + new_data = self._mgr.take( indices, axis=self._get_block_manager_axis(axis), verify=True ) @@ -3506,6 +3508,8 @@ class animal locomotion if axis == 1: return self[key] + self._consolidate_inplace() + index = self.index if isinstance(index, MultiIndex): loc, new_index = self.index.get_loc_level(key, drop_level=drop_level) @@ -5428,6 +5432,7 @@ def values(self) -> np.ndarray: ['lion', 80.5, 1], ['monkey', nan, None]], dtype=object) """ + self._consolidate_inplace() return self._mgr.as_array(transpose=self._AXIS_REVERSED) @property @@ -6096,6 +6101,8 @@ def fillna( inplace = validate_bool_kwarg(inplace, "inplace") value, method = validate_fillna_kwargs(value, method) + self._consolidate_inplace() + # set the default here, so functions examining the signaure # can detect if something was set (e.g. in groupby) (GH9221) if axis is None: @@ -6530,6 +6537,8 @@ def replace( if not is_bool(regex) and to_replace is not None: raise AssertionError("'to_replace' must be 'None' if 'regex' is not a bool") + self._consolidate_inplace() + if value is None: # passing a single value that is scalar like # when value is None (GH5319), for compat diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index e43c7bf887bcc..6a7df1c8d88a1 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -378,6 +378,8 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T: result_blocks: List[Block] = [] # fillna: Series/DataFrame is responsible for making sure value is aligned + self._consolidate_inplace() + aligned_args = {k: kwargs[k] for k in align_keys} for b in self.blocks: @@ -410,6 +412,7 @@ def apply(self: T, f, align_keys=None, **kwargs) -> T: def quantile( self, axis: int = 0, + consolidate: bool = True, transposed: bool = False, interpolation="linear", qs=None, @@ -423,6 +426,8 @@ def quantile( Parameters ---------- axis: reduction axis, default 0 + consolidate: bool, default True. Join together blocks having same + dtype transposed: bool, default False we are holding transposed data interpolation : type of interpolation, default 'linear' @@ -437,6 +442,9 @@ def quantile( # simplify some of the code here and in the blocks assert self.ndim >= 2 + if consolidate: + self._consolidate_inplace() + def get_axe(block, qs, axes): # Because Series dispatches to DataFrame, we will always have # block.ndim == 2 @@ -667,6 +675,8 @@ def is_mixed_type(self) -> bool: @property def is_numeric_mixed_type(self) -> bool: + # Warning, consolidation needs to get checked upstairs + self._consolidate_inplace() return all(block.is_numeric for block in self.blocks) @property From 1228510e120e98320fc0a6a566d4051df9347631 Mon Sep 17 00:00:00 2001 From: Joris Van den Bossche Date: Wed, 10 Jun 2020 10:01:12 +0200 Subject: [PATCH 2/5] re-revert take and numeric_mixed_type --- pandas/core/generic.py | 2 -- pandas/core/internals/managers.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 8aa8f8bb60654..54bf76990ff03 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3369,8 +3369,6 @@ class max_speed nv.validate_take(tuple(), kwargs) - self._consolidate_inplace() - new_data = self._mgr.take( indices, axis=self._get_block_manager_axis(axis), verify=True ) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 6a7df1c8d88a1..199148e02ee23 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -675,8 +675,6 @@ def is_mixed_type(self) -> bool: @property def is_numeric_mixed_type(self) -> bool: - # Warning, consolidation needs to get checked upstairs - self._consolidate_inplace() return all(block.is_numeric for block in self.blocks) @property From 96e804b43f01fa86d9a531a3839113019b28f74f Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 13:40:08 +0100 Subject: [PATCH 3/5] release note --- doc/source/whatsnew/v1.1.3.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v1.1.3.rst b/doc/source/whatsnew/v1.1.3.rst index eded30ca45025..0f908de41a7bb 100644 --- a/doc/source/whatsnew/v1.1.3.rst +++ b/doc/source/whatsnew/v1.1.3.rst @@ -34,6 +34,7 @@ Fixed regressions - Fixed regression when adding a :meth:`timedelta_range` to a :class:``Timestamp`` raised an ``ValueError`` (:issue:`35897`) - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a tuple (:issue:`35534`) - Fixed regression in :meth:`Series.__getitem__` incorrectly raising when the input was a frozenset (:issue:`35747`) +- Fixed regression where :meth:`DataFrame.fillna` not filling ``NaN`` after :meth:`DataFrame.pivot` operation (:issue:`36495`) - Fixed regression in :meth:`read_excel` with ``engine="odf"`` caused ``UnboundLocalError`` in some cases where cells had nested child nodes (:issue:`36122`, :issue:`35802`) - Fixed regression in :meth:`DataFrame.replace` inconsistent replace when using a float in the replace method (:issue:`35376`) - Fixed regression in :class:`DataFrame` and :class:`Series` comparisons between numeric arrays and strings (:issue:`35700`, :issue:`36377`) From 90a1d6fb52d112fa2ba68a3bd02a44ebe1f02e38 Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 13:56:23 +0100 Subject: [PATCH 4/5] add test --- pandas/tests/frame/test_missing.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pandas/tests/frame/test_missing.py b/pandas/tests/frame/test_missing.py index 5d3f8e3a2f7c1..d3cef29e78980 100644 --- a/pandas/tests/frame/test_missing.py +++ b/pandas/tests/frame/test_missing.py @@ -718,3 +718,31 @@ def test_fill_corner(self, float_frame, float_string_frame): # TODO(wesm): unused? result = empty_float.fillna(value=0) # noqa + + def test_fillna_after_pivot(self): + # https://github.com/pandas-dev/pandas/issues/36495 + df = DataFrame( + [ + [1, 1, 1, 1.0], + [2, 2, 2, 2.0], + [3, 3, 3, 3.0], + ], + columns=["i1", "i2", "i3", "f1"], + ) + + result = df.pivot("i1", "i2").fillna(0) + + expected = DataFrame( + [ + [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], + [0.0, 2.0, 0.0, 0.0, 2.0, 0.0], + [0.0, 0.0, 3.0, 0.0, 0.0, 3.0], + ], + index=pd.Int64Index([1, 2, 3], dtype="int64", name="i1"), + columns=pd.MultiIndex.from_tuples( + [("i3", 1), ("i3", 2), ("i3", 3), ("f1", 1), ("f1", 2), ("f1", 3)], + names=[None, "i2"], + ), + ) + + tm.assert_frame_equal(result, expected) From d164b49ab2e6a750caf2640320b5ec2e0adc0d1c Mon Sep 17 00:00:00 2001 From: Simon Hawkins Date: Sat, 26 Sep 2020 14:17:20 +0100 Subject: [PATCH 5/5] minimise changes --- pandas/core/generic.py | 6 ------ pandas/core/internals/managers.py | 6 ------ 2 files changed, 12 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index c861101a769cf..bd720151fb15e 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -3662,8 +3662,6 @@ class animal locomotion if axis == 1: return self[key] - self._consolidate_inplace() - index = self.index if isinstance(index, MultiIndex): try: @@ -6247,8 +6245,6 @@ def fillna( inplace = validate_bool_kwarg(inplace, "inplace") value, method = validate_fillna_kwargs(value, method) - self._consolidate_inplace() - # set the default here, so functions examining the signaure # can detect if something was set (e.g. in groupby) (GH9221) if axis is None: @@ -6670,8 +6666,6 @@ def replace( if not is_bool(regex) and to_replace is not None: raise ValueError("'to_replace' must be 'None' if 'regex' is not a bool") - self._consolidate_inplace() - if value is None: # passing a single value that is scalar like # when value is None (GH5319), for compat diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 854c4b2651e01..e5a5f1f3661e6 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -414,7 +414,6 @@ def apply( def quantile( self, axis: int = 0, - consolidate: bool = True, transposed: bool = False, interpolation="linear", qs=None, @@ -428,8 +427,6 @@ def quantile( Parameters ---------- axis: reduction axis, default 0 - consolidate: bool, default True. Join together blocks having same - dtype transposed: bool, default False we are holding transposed data interpolation : type of interpolation, default 'linear' @@ -444,9 +441,6 @@ def quantile( # simplify some of the code here and in the blocks assert self.ndim >= 2 - if consolidate: - self._consolidate_inplace() - def get_axe(block, qs, axes): # Because Series dispatches to DataFrame, we will always have # block.ndim == 2