Merge branch 'main' into ref-split-pattern

Author: jreback

.github/workflows/posix.yml

@@ -13,7 +13,6 @@ on:
       - "doc/**"
 
 env:
-  PYTEST_WORKERS: "auto"
   PANDAS_CI: 1
 
 jobs:
@@ -25,33 +24,48 @@ jobs:
     timeout-minutes: 120
     strategy:
       matrix:
-        settings: [
-          [actions-38-downstream_compat.yaml, "not slow and not network and not single_cpu", "", "", "", "", ""],
-          [actions-38-minimum_versions.yaml, "not single_cpu", "", "", "", "", ""],
-          [actions-38.yaml, "not slow and not network and not single_cpu", "language-pack-it", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network and not single_cpu", "language-pack-zh-hans", "zh_CN.utf8", "zh_CN.utf8", "", ""],
-          [actions-38.yaml, "not single_cpu", "", "", "", "", ""],
-          [actions-pypy-38.yaml, "not slow and not single_cpu", "", "", "", "", "--max-worker-restart 0"],
-          [actions-39.yaml, "not single_cpu", "", "", "", "", ""],
-          [actions-310-numpydev.yaml, "not slow and not network and not single_cpu", "", "", "", "deprecate", "-W error"],
-          [actions-310.yaml, "not single_cpu", "", "", "", "", ""],
-        ]
+        env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml]
+        pattern: ["not single_cpu", "single_cpu"]
+        include:
+          - env_file: actions-38-downstream_compat.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pytest_target: "pandas/tests/test_downstream.py"
+          - env_file: actions-38-minimum_versions.yaml
+            pattern: "not slow and not network and not single_cpu"
+          - env_file: actions-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            extra_apt: "language-pack-it"
+            lang: "it_IT.utf8"
+            lc_all: "it_IT.utf8"
+          - env_file: actions-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            extra_apt: "language-pack-zh-hans"
+            lang: "zh_CN.utf8"
+            lc_all: "zh_CN.utf8"
+          - env_file: actions-pypy-38.yaml
+            pattern: "not slow and not network and not single_cpu"
+            test_args: "--max-worker-restart 0"
+          - env_file: actions-310-numpydev.yaml
+            pattern: "not slow and not network and not single_cpu"
+            pandas_testing_mode: "deprecate"
+            test_args: "-W error"
       fail-fast: false
     env:
-      ENV_FILE: ci/deps/${{ matrix.settings[0] }}
-      PATTERN: ${{ matrix.settings[1] }}
-      EXTRA_APT: ${{ matrix.settings[2] }}
-      LANG: ${{ matrix.settings[3] }}
-      LC_ALL: ${{ matrix.settings[4] }}
-      PANDAS_TESTING_MODE: ${{ matrix.settings[5] }}
-      TEST_ARGS: ${{ matrix.settings[6] }}
-      PYTEST_TARGET:  pandas
-      IS_PYPY: ${{ contains(matrix.settings[0], 'pypy') }}
+      ENV_FILE: ci/deps/${{ matrix.env_file }}
+      PATTERN: ${{ matrix.pattern }}
+      EXTRA_APT: ${{ matrix.extra_apt || '' }}
+      LANG: ${{ matrix.lang || '' }}
+      LC_ALL: ${{ matrix.lc_all || '' }}
+      PANDAS_TESTING_MODE: ${{ matrix.pandas_testing_mode || '' }}
+      TEST_ARGS: ${{ matrix.test_args || '' }}
+      PYTEST_WORKERS: ${{ contains(matrix.pattern, 'not single_cpu') && 'auto' || '1' }}
+      PYTEST_TARGET: ${{ matrix.pytest_target || 'pandas' }}
+      IS_PYPY: ${{ contains(matrix.env_file, 'pypy') }}
       # TODO: re-enable coverage on pypy, its slow
-      COVERAGE: ${{ !contains(matrix.settings[0], 'pypy') }}
+      COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}-${{ matrix.settings[1] }}-${{ matrix.settings[2] }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}
       cancel-in-progress: true
 
     services:
@@ -129,8 +143,7 @@ jobs:
       shell: bash
       run: |
         # TODO: re-enable cov, its slowing the tests down though
-        # TODO: Unpin Cython, the new Cython 0.29.26 is causing compilation errors
-        pip install Cython==0.29.25 numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
+        pip install Cython numpy python-dateutil pytz pytest>=6.0 pytest-xdist>=1.31.0 hypothesis>=5.5.3
       if: ${{ env.IS_PYPY == 'true' }}
 
     - name: Build Pandas

doc/source/user_guide/io.rst

@@ -3675,6 +3675,10 @@ should be passed to ``index_col`` and ``header``:
 
    os.remove("path_to_file.xlsx")
 
+Missing values in columns specified in ``index_col`` will be forward filled to
+allow roundtripping with ``to_excel`` for ``merged_cells=True``. To avoid forward
+filling the missing values use ``set_index`` after reading the data instead of
+``index_col``.
 
 Parsing specific columns
 ++++++++++++++++++++++++

doc/source/whatsnew/v1.4.2.rst

@@ -14,7 +14,7 @@ including other versions of pandas.
 
 Fixed regressions
 ~~~~~~~~~~~~~~~~~
--
+- Fixed regression in :meth:`DataFrame.drop` and :meth:`Series.drop` when :class:`Index` had extension dtype and duplicates (:issue:`45820`)
 -
 
 .. ---------------------------------------------------------------------------

doc/source/whatsnew/v1.5.0.rst

@@ -304,7 +304,7 @@ Conversion
 
 Strings
 ^^^^^^^
--
+- Bug in :meth:`str.startswith` and :meth:`str.endswith` when using other series as parameter _pat_. Now raises ``TypeError`` (:issue:`3485`)
 -
 
 Interval
@@ -316,6 +316,7 @@ Indexing
 ^^^^^^^^
 - Bug in :meth:`loc.__getitem__` with a list of keys causing an internal inconsistency that could lead to a disconnect between ``frame.at[x, y]`` vs ``frame[y].loc[x]`` (:issue:`22372`)
 - Bug in :meth:`DataFrame.iloc` where indexing a single row on a :class:`DataFrame` with a single ExtensionDtype column gave a copy instead of a view on the underlying data (:issue:`45241`)
+- Bug in :meth:`Series.align` does not create :class:`MultiIndex` with union of levels when both MultiIndexes intersections are identical (:issue:`45224`)
 - Bug in setting a NA value (``None`` or ``np.nan``) into a :class:`Series` with int-based :class:`IntervalDtype` incorrectly casting to object dtype instead of a float-based :class:`IntervalDtype` (:issue:`45568`)
 - Bug in :meth:`Series.__setitem__` with a non-integer :class:`Index` when using an integer key to set a value that cannot be set inplace where a ``ValueError`` was raised instead of casting to a common dtype (:issue:`45070`)
 - Bug in :meth:`Series.__setitem__` when setting incompatible values into a ``PeriodDtype`` or ``IntervalDtype`` :class:`Series` raising when indexing with a boolean mask but coercing when indexing with otherwise-equivalent indexers; these now consistently coerce, along with :meth:`Series.mask` and :meth:`Series.where` (:issue:`45768`)
@@ -326,6 +327,7 @@ Indexing
 - Bug in :meth:`Series.__setitem__` when setting ``boolean`` dtype values containing ``NA`` incorrectly raising instead of casting to ``boolean`` dtype (:issue:`45462`)
 - Bug in :meth:`Series.__setitem__` where setting :attr:`NA` into a numeric-dtpye :class:`Series` would incorrectly upcast to object-dtype rather than treating the value as ``np.nan`` (:issue:`44199`)
 - Bug in :meth:`Series.__setitem__` with ``datetime64[ns]`` dtype, an all-``False`` boolean mask, and an incompatible value incorrectly casting to ``object`` instead of retaining ``datetime64[ns]`` dtype (:issue:`45967`)
+- Bug in :meth:`Index.__getitem__`  raising ``ValueError`` when indexer is from boolean dtype with ``NA`` (:issue:`45806`)
 - Bug in :meth:`Series.mask` with ``inplace=True`` or setting values with a boolean mask with small integer dtypes incorrectly raising (:issue:`45750`)
 - Bug in :meth:`DataFrame.mask` with ``inplace=True`` and ``ExtensionDtype`` columns incorrectly raising (:issue:`45577`)
 - Bug in getting a column from a DataFrame with an object-dtype row index with datetime-like values: the resulting Series now preserves the exact object-dtype Index from the parent DataFrame (:issue:`42950`)
@@ -353,12 +355,13 @@ I/O
 - Bug in :func:`read_excel` results in an infinite loop with certain ``skiprows`` callables (:issue:`45585`)
 - Bug in :meth:`DataFrame.info` where a new line at the end of the output is omitted when called on an empty :class:`DataFrame` (:issue:`45494`)
 - Bug in :func:`read_csv` not recognizing line break for ``on_bad_lines="warn"`` for ``engine="c"`` (:issue:`41710`)
+- Bug in :meth:`DataFrame.to_csv` not respecting ``float_format`` for ``Float64`` dtype (:issue:`45991`)
 - Bug in :func:`read_parquet` when ``engine="pyarrow"`` which caused partial write to disk when column of unsupported datatype was passed (:issue:`44914`)
 - Bug in :func:`DataFrame.to_excel` and :class:`ExcelWriter` would raise when writing an empty DataFrame to a ``.ods`` file (:issue:`45793`)
 
 Period
 ^^^^^^
--
+- Bug in subtraction of :class:`Period` from :class:`PeriodArray` returning wrong results (:issue:`45999`)
 -
 
 Plotting
@@ -372,7 +375,8 @@ Plotting
 Groupby/resample/rolling
 ^^^^^^^^^^^^^^^^^^^^^^^^
 - Bug in :meth:`DataFrame.resample` ignoring ``closed="right"`` on :class:`TimedeltaIndex` (:issue:`45414`)
-- Bug in :meth:`.DataFrameGroupBy.transform` fails when the input DataFrame has multiple columns (:issue:`27469`)
+- Bug in :meth:`.DataFrameGroupBy.transform` fails when ``func="size"`` and the input DataFrame has multiple columns (:issue:`27469`)
+- Bug in :meth:`.DataFrameGroupBy.size` and :meth:`.DataFrameGroupBy.transform` with ``func="size"`` produced incorrect results when ``axis=1`` (:issue:`45715`)
 
 Reshaping
 ^^^^^^^^^

pandas/_libs/tslibs/ctime.pyx

@@ -0,0 +1,78 @@
+"""
+Cython implementation of (parts of) the standard library time module.
+"""
+
+from cpython.exc cimport PyErr_SetFromErrno
+from libc.stdint cimport int64_t
+
+
+cdef extern from "Python.h":
+    ctypedef int64_t _PyTime_t
+    _PyTime_t _PyTime_GetSystemClock() nogil
+    double _PyTime_AsSecondsDouble(_PyTime_t t) nogil
+
+from libc.time cimport (
+    localtime as libc_localtime,
+    time_t,
+    tm,
+)
+
+
+def pytime():
+    """
+    python-exposed for testing
+    """
+    return time()
+
+
+def pylocaltime():
+    """
+    python-exposed for testing
+    """
+    lt = localtime()
+    # https://github.com/pandas-dev/pandas/pull/45864#issuecomment-1033021599
+    return {
+        "tm_year": lt.tm_year,
+        "tm_mon": lt.tm_mon,
+        "tm_mday": lt.tm_mday,
+        "tm_hour": lt.tm_hour,
+        "tm_min": lt.tm_min,
+        "tm_sec": lt.tm_sec,
+        "tm_wday": lt.tm_wday,
+        "tm_yday": lt.tm_yday,
+        "tm_isdst": lt.tm_isdst,
+    }
+
+
+cdef inline double time() nogil:
+    cdef:
+        _PyTime_t tic
+
+    tic = _PyTime_GetSystemClock()
+    return _PyTime_AsSecondsDouble(tic)
+
+
+cdef inline int _raise_from_errno() except -1 with gil:
+    PyErr_SetFromErrno(RuntimeError)
+    return <int>-1  # Let the C compiler know that this function always raises.
+
+
+cdef inline tm localtime() nogil except *:
+    """
+    Analogue to the stdlib time.localtime.  The returned struct
+    has some entries that the stdlib version does not: tm_gmtoff, tm_zone
+    """
+    cdef:
+        time_t tic = <time_t>time()
+        tm* result
+
+    result = libc_localtime(&tic)
+    if result is NULL:
+        _raise_from_errno()
+    # Fix 0-based date values (and the 1900-based year).
+    # See tmtotuple() in https://github.com/python/cpython/blob/master/Modules/timemodule.c
+    result.tm_year += 1900
+    result.tm_mon += 1
+    result.tm_wday = (result.tm_wday + 6) % 7
+    result.tm_yday += 1
+    return result[0]

pandas/_testing/_io.py

@@ -3,6 +3,7 @@
 import bz2
 from functools import wraps
 import gzip
+import socket
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -73,7 +74,13 @@ def _get_default_network_errors():
     import http.client
     import urllib.error
 
-    return (OSError, http.client.HTTPException, TimeoutError, urllib.error.URLError)
+    return (
+        OSError,
+        http.client.HTTPException,
+        TimeoutError,
+        urllib.error.URLError,
+        socket.timeout,
+    )
 
 
 def optional_args(decorator):
@@ -264,8 +271,10 @@ def can_connect(url, error_classes=None):
         error_classes = _get_default_network_errors()
 
     try:
-        with urlopen(url):
-            pass
+        with urlopen(url, timeout=20) as response:
+            # Timeout just in case rate-limiting is applied
+            if response.status != 200:
+                return False
     except error_classes:
         return False
     else:

pandas/_testing/contexts.py

@@ -3,14 +3,13 @@
 from contextlib import contextmanager
 import os
 from pathlib import Path
-import random
 from shutil import rmtree
-import string
 import tempfile
 from typing import (
     IO,
     Any,
 )
+import uuid
 
 import numpy as np
 
@@ -107,9 +106,7 @@ def ensure_clean(filename=None, return_filelike: bool = False, **kwargs: Any):
 
     if filename is None:
         filename = ""
-    filename = (
-        "".join(random.choices(string.ascii_letters + string.digits, k=30)) + filename
-    )
+    filename = str(uuid.uuid4()) + filename
     path = folder / filename
 
     path.touch()

pandas/conftest.py

@@ -229,6 +229,14 @@ def axis(request):
 axis_frame = axis
 
 
+@pytest.fixture(params=[1, "columns"], ids=lambda x: f"axis={repr(x)}")
+def axis_1(request):
+    """
+    Fixture for returning aliases of axis 1 of a DataFrame.
+    """
+    return request.param
+
+
 @pytest.fixture(params=[True, False, None])
 def observed(request):
     """

pandas/core/algorithms.py

@@ -64,6 +64,7 @@
 )
 from pandas.core.dtypes.concat import concat_compat
 from pandas.core.dtypes.dtypes import (
+    BaseMaskedDtype,
     ExtensionDtype,
     PandasDtype,
 )
@@ -103,6 +104,7 @@
         Series,
     )
     from pandas.core.arrays import (
+        BaseMaskedArray,
         DatetimeArray,
         ExtensionArray,
         TimedeltaArray,
@@ -142,6 +144,15 @@ def _ensure_data(values: ArrayLike) -> np.ndarray:
     if is_object_dtype(values.dtype):
         return ensure_object(np.asarray(values))
 
+    elif isinstance(values.dtype, BaseMaskedDtype):
+        # i.e. BooleanArray, FloatingArray, IntegerArray
+        values = cast("BaseMaskedArray", values)
+        if not values._hasna:
+            # No pd.NAs -> We can avoid an object-dtype cast (and copy) GH#41816
+            #  recurse to avoid re-implementing logic for eg bool->uint8
+            return _ensure_data(values._data)
+        return np.asarray(values)
+
     elif is_bool_dtype(values.dtype):
         if isinstance(values, np.ndarray):
             # i.e. actually dtype == np.dtype("bool")
@@ -1188,18 +1199,6 @@ def compute(self, method: str) -> Series:
         dropped = self.obj.dropna()
         nan_index = self.obj.drop(dropped.index)
 
-        if is_extension_array_dtype(dropped.dtype):
-            # GH#41816 bc we have dropped NAs above, MaskedArrays can use the
-            #  numpy logic.
-            from pandas.core.arrays import BaseMaskedArray
-
-            arr = dropped._values
-            if isinstance(arr, BaseMaskedArray):
-                ser = type(dropped)(arr._data, index=dropped.index, name=dropped.name)
-
-                result = type(self)(ser, n=self.n, keep=self.keep).compute(method)
-                return result.astype(arr.dtype)
-
         # slow method
         if n >= len(self.obj):
             ascending = method == "nsmallest"

pandas/core/arrays/boolean.py

@@ -381,6 +381,3 @@ def _logical_method(self, other, op):
         # error: Argument 2 to "BooleanArray" has incompatible type "Optional[Any]";
         # expected "ndarray"
         return BooleanArray(result, mask)  # type: ignore[arg-type]
-
-    def __abs__(self):
-        return self.copy()