Merge branch 'master' into take-error

Author: gfkang

.github/workflows/ci.yml

@@ -23,8 +23,9 @@ jobs:
         shell: bash -l {0}
 
     concurrency:
-      group: ${{ github.ref }}-checks
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-checks
+      cancel-in-progress: true
 
     steps:
     - name: Checkout
@@ -102,7 +103,8 @@ jobs:
     runs-on: ubuntu-latest
 
     concurrency:
-      group: ${{ github.ref }}-web-docs
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs
       cancel-in-progress: true
 
     steps:
@@ -160,11 +162,20 @@ jobs:
   data_manager:
     name: Test experimental data manager
     runs-on: ubuntu-latest
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
     strategy:
       matrix:
         pattern: ["not slow and not network and not clipboard", "slow"]
     concurrency:
-      group: ${{ github.ref }}-data_manager-${{ matrix.pattern }}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-data_manager-${{ matrix.pattern }}
       cancel-in-progress: true
 
     steps:

.github/workflows/database.yml

@@ -31,8 +31,9 @@ jobs:
       fail-fast: false
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.ENV_FILE }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.ENV_FILE }}
+      cancel-in-progress: true
 
     services:
       mysql:
@@ -62,6 +63,14 @@ jobs:
         ports:
           - 5432:5432
 
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
+
     steps:
     - name: Checkout
       uses: actions/checkout@v2

.github/workflows/posix.yml

@@ -27,12 +27,12 @@ jobs:
         settings: [
           [actions-38-minimum_versions.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
           [actions-38-locale_slow.yaml, "slow", "language-pack-it xsel", "it_IT.utf8", "it_IT.utf8", "", ""],
-          [actions-38.yaml, "not slow and not network and not clipboard", "", "", "", "", ""],
+          [actions-38.yaml, "not slow and not clipboard", "", "", "", "", ""],
           [actions-38-slow.yaml, "slow", "", "", "", "", ""],
           [actions-38-locale.yaml, "not slow and not network", "language-pack-zh-hans xsel", "zh_CN.utf8", "zh_CN.utf8", "", ""],
           [actions-39-slow.yaml, "slow", "", "", "", "", ""],
           [actions-39-numpydev.yaml, "not slow and not network", "xsel", "", "", "deprecate", "-W error"],
-          [actions-39.yaml, "not slow and not network and not clipboard", "", "", "", "", ""]
+          [actions-39.yaml, "not slow and not clipboard", "", "", "", "", ""]
         ]
       fail-fast: false
     env:
@@ -46,8 +46,17 @@ jobs:
       TEST_ARGS: ${{ matrix.settings[6] }}
       PYTEST_TARGET:  pandas
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.settings[0] }}
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.settings[0] }}
+      cancel-in-progress: true
+    services:
+      moto:
+        image: motoserver/moto
+        env:
+          AWS_ACCESS_KEY_ID: foobar_key
+          AWS_SECRET_ACCESS_KEY: foobar_secret
+        ports:
+          - 5000:5000
 
     steps:
     - name: Checkout

.github/workflows/pre-commit.yml

@@ -11,8 +11,9 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     concurrency:
-      group: ${{ github.ref }}-pre-commit
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pre-commit
+      cancel-in-progress: true
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

.github/workflows/python-dev.yml

@@ -31,8 +31,9 @@ jobs:
     timeout-minutes: 80
 
     concurrency:
-      group: ${{ github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      #https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.os }}-${{ matrix.pytest_target }}-dev
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/sdist.yml

@@ -25,8 +25,9 @@ jobs:
       matrix:
         python-version: ["3.8", "3.9", "3.10"]
     concurrency:
-      group: ${{github.ref}}-${{matrix.python-version}}-sdist
-      cancel-in-progress: ${{github.event_name == 'pull_request'}}
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{matrix.python-version}}-sdist
+      cancel-in-progress: true
 
     steps:
     - uses: actions/checkout@v2

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
     -   id: absolufy-imports
         files: ^pandas/
 -   repo: https://github.com/python/black
-    rev: 21.10b0
+    rev: 21.11b1
     hooks:
     -   id: black
 -   repo: https://github.com/codespell-project/codespell
@@ -55,11 +55,11 @@ repos:
         types: [text]
         args: [--append-config=flake8/cython-template.cfg]
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.9.3
+    rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.0
+    rev: v2.29.1
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -111,6 +111,8 @@ repos:
             # Incorrect code-block / IPython directives
             |\.\.\ code-block\ ::
             |\.\.\ ipython\ ::
+            # directive should not have a space before ::
+            |\.\.\ \w+\ ::
 
             # Check for deprecated messages without sphinx directive
             |(DEPRECATED|DEPRECATE|Deprecated)(:|,|\.)

asv_bench/benchmarks/frame_ctor.py

@@ -19,7 +19,10 @@
     )
 except ImportError:
     # For compatibility with older versions
-    from pandas.core.datetools import *  # noqa
+    from pandas.core.datetools import (
+        Hour,
+        Nano,
+    )
 
 
 class FromDicts:

asv_bench/benchmarks/frame_methods.py

@@ -76,7 +76,7 @@ def time_reindex_axis1_missing(self):
         self.df.reindex(columns=self.idx)
 
     def time_reindex_both_axes(self):
-        self.df.reindex(index=self.idx, columns=self.idx)
+        self.df.reindex(index=self.idx, columns=self.idx_cols)
 
     def time_reindex_upcast(self):
         self.df2.reindex(np.random.permutation(range(1200)))
@@ -731,17 +731,6 @@ def time_dataframe_describe(self):
         self.df.describe()
 
 
-class SelectDtypes:
-    params = [100, 1000]
-    param_names = ["n"]
-
-    def setup(self, n):
-        self.df = DataFrame(np.random.randn(10, n))
-
-    def time_select_dtypes(self, n):
-        self.df.select_dtypes(include="int")
-
-
 class MemoryUsage:
     def setup(self):
         self.df = DataFrame(np.random.randn(100000, 2), columns=list("AB"))

asv_bench/benchmarks/groupby.py

@@ -443,7 +443,7 @@ class GroupByMethods:
             "var",
         ],
         ["direct", "transformation"],
-        [1, 2, 5, 10],
+        [1, 5],
     ]
 
     def setup(self, dtype, method, application, ncols):
@@ -455,6 +455,7 @@ def setup(self, dtype, method, application, ncols):
             raise NotImplementedError
 
         if application == "transformation" and method in [
+            "describe",
             "head",
             "tail",
             "unique",
@@ -464,7 +465,12 @@ def setup(self, dtype, method, application, ncols):
             # DataFrameGroupBy doesn't have these methods
             raise NotImplementedError
 
-        ngroups = 1000
+        if method == "describe":
+            ngroups = 20
+        elif method in ["mad", "skew"]:
+            ngroups = 100
+        else:
+            ngroups = 1000
         size = ngroups * 2
         rng = np.arange(ngroups).reshape(-1, 1)
         rng = np.broadcast_to(rng, (len(rng), ncols))
@@ -491,9 +497,6 @@ def setup(self, dtype, method, application, ncols):
             cols = cols[0]
 
         if application == "transformation":
-            if method == "describe":
-                raise NotImplementedError
-
             self.as_group_method = lambda: df.groupby("key")[cols].transform(method)
             self.as_field_method = lambda: df.groupby(cols)["key"].transform(method)
         else: