merge main

Author: ntachukwu

.github/workflows/32-bit-linux.yml

@@ -38,7 +38,8 @@ jobs:
           /opt/python/cp38-cp38/bin/python -m venv ~/virtualenvs/pandas-dev && \
           . ~/virtualenvs/pandas-dev/bin/activate && \
           python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \
-          pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
+          python -m pip install versioneer[toml] && \
+          python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \
           python setup.py build_ext -q -j1 && \
           python -m pip install --no-build-isolation --no-use-pep517 -e . && \
           python -m pip list && \

.github/workflows/code-checks.yml

@@ -37,8 +37,8 @@ jobs:
     - name: Run pre-commit
       uses: pre-commit/[email protected]
 
-  typing_and_docstring_validation:
-    name: Docstring and typing validation
+  docstring_typing_pylint:
+    name: Docstring validation, typing, and pylint
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -79,6 +79,10 @@ jobs:
       run: ci/code_checks.sh docstrings
       if: ${{ steps.build.outcome == 'success' && always() }}
 
+    - name: Run check of documentation notebooks
+      run: ci/code_checks.sh notebooks
+      if: ${{ steps.build.outcome == 'success' && always() }}
+
     - name: Use existing environment for type checking
       run: |
         echo $PATH >> $GITHUB_PATH

.github/workflows/docbuild-and-upload.yml

@@ -54,32 +54,27 @@ jobs:
     - name: Build documentation zip
       run: doc/make.py zip_html
 
-    - name: Build the interactive terminal
-      run: |
-        cd web/interactive_terminal
-        jupyter lite build
-
     - name: Install ssh key
       run: |
         mkdir -m 700 -p ~/.ssh
         echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa
         chmod 600 ~/.ssh/id_rsa
-        echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts
+        echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFjYkJBk7sos+r7yATODogQc3jUdW1aascGpyOD4bohj8dWjzwLJv/OJ/fyOQ5lmj81WKDk67tGtqNJYGL9acII=" > ~/.ssh/known_hosts
       if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
 
     - name: Copy cheatsheets into site directory
       run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/
 
     - name: Upload web
-      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas
+      run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ web@${{ secrets.server_ip }}:/var/www/html
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload dev docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev
+      run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/dev
       if: github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     - name: Upload prod docs
-      run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/version/${GITHUB_REF_NAME:1}
+      run: rsync -az --delete doc/build/html/ web@${{ secrets.server_ip }}:/var/www/html/pandas-docs/version/${GITHUB_REF_NAME:1}
       if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
 
     - name: Move docs into site directory

.github/workflows/package-checks.yml

@@ -0,0 +1,52 @@
+name: Package Checks
+
+on:
+  push:
+    branches:
+      - main
+      - 1.5.x
+  pull_request:
+    branches:
+      - main
+      - 1.5.x
+    types: [ labeled, opened, synchronize, reopened ]
+
+permissions:
+  contents: read
+
+jobs:
+  pip:
+    if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        extra: ["test", "performance", "timezone", "computation", "fss", "aws", "gcp", "excel", "parquet", "feather", "hdf5", "spss", "postgresql", "mysql", "sql-other", "html", "xml", "plot", "output_formatting", "clipboard", "compression", "all"]
+      fail-fast: false
+    name: Install Extras - ${{ matrix.extra }}
+    concurrency:
+      # https://github.community/t/concurrecy-not-work-for-push/183068/7
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-pip-extras-${{ matrix.extra }}
+      cancel-in-progress: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Setup Python
+        id: setup_python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.8'
+
+      - name: Install required dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython
+          python -m pip install versioneer[toml]
+        shell: bash -el {0}
+
+      - name: Pip install with extra
+        run: |
+          python -m pip install -e .[${{ matrix.extra }}] --no-build-isolation
+        shell: bash -el {0}

.github/workflows/python-dev.yml

@@ -75,6 +75,7 @@ jobs:
         python -m pip install --upgrade pip setuptools wheel
         python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy
         python -m pip install git+https://github.com/nedbat/coveragepy.git
+        python -m pip install versioneer[toml]
         python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17
         python -m pip list
 

.github/workflows/sdist.yml

@@ -47,6 +47,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
+        python -m pip install versioneer[toml]
 
         # GH 39416
         pip install numpy

.pre-commit-config.yaml

@@ -26,12 +26,13 @@ repos:
     hooks:
     -   id: codespell
         types_or: [python, rst, markdown]
+        additional_dependencies: [tomli]
 -   repo: https://github.com/MarcoGorelli/cython-lint
-    rev: v0.2.1
+    rev: v0.9.1
     hooks:
     -   id: cython-lint
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v4.4.0
     hooks:
     -   id: debug-statements
     -   id: end-of-file-fixer
@@ -50,26 +51,45 @@ repos:
         exclude: ^pandas/_libs/src/(klib|headers)/
         args: [--quiet, '--extensions=c,h', '--headers=h', --recursive, '--filter=-readability/casting,-runtime/int,-build/include_subdir']
 -   repo: https://github.com/PyCQA/flake8
-    rev: 5.0.4
+    rev: 6.0.0
     hooks:
     -   id: flake8
         # Need to patch os.remove rule in pandas-dev-flaker
         exclude: ^ci/fix_wheels.py
         additional_dependencies: &flake8_dependencies
-        - flake8==5.0.4
+        - flake8==6.0.0
         - flake8-bugbear==22.7.1
         - pandas-dev-flaker==0.5.0
 -   repo: https://github.com/pycqa/pylint
-    rev: v2.15.5
+    rev: v2.15.6
     hooks:
     -   id: pylint
         stages: [manual]
+-   repo: https://github.com/pycqa/pylint
+    rev: v2.15.6
+    hooks:
+    -   id: pylint
+        alias: redefined-outer-name
+        name: Redefining name from outer scope
+        files: ^pandas/
+        exclude: |
+            (?x)
+            ^pandas/tests  # keep excluded
+            |/_testing/  # keep excluded
+            |^pandas/util/_test_decorators\.py  # keep excluded
+            |^pandas/_version\.py  # keep excluded
+            |^pandas/conftest\.py  # keep excluded
+            |^pandas/core/tools/datetimes\.py
+            |^pandas/io/formats/format\.py
+            |^pandas/core/generic\.py
+        args: [--disable=all, --enable=redefined-outer-name]
+        stages: [manual]
 -   repo: https://github.com/PyCQA/isort
     rev: 5.10.1
     hooks:
     -   id: isort
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.0
+    rev: v3.2.2
     hooks:
     -   id: pyupgrade
         args: [--py38-plus]
@@ -201,7 +221,7 @@ repos:
         entry: python scripts/sync_flake8_versions.py
         files: ^(\.pre-commit-config\.yaml|environment\.yml)$
         pass_filenames: false
-        additional_dependencies: [pyyaml]
+        additional_dependencies: [pyyaml, toml]
     -   id: title-capitalization
         name: Validate correct capitalization among titles in documentation
         entry: python scripts/validate_rst_title_capitalization.py
@@ -287,5 +307,4 @@ repos:
         language: python
         additional_dependencies:
         - autotyping==22.9.0
-        - black==22.6.0
         - libcst==0.4.7

asv_bench/asv.conf.json

@@ -57,7 +57,7 @@
         "odfpy": [],
         "jinja2": [],
     },
-    "conda_channels": ["defaults", "conda-forge"],
+    "conda_channels": ["conda-forge"],
     // Combinations of libraries/python versions can be excluded/included
     // from the set to test. Each entry is a dictionary containing additional
     // key-value pairs to include/exclude.
@@ -125,6 +125,7 @@
     "regression_thresholds": {
     },
     "build_command":
-    ["python setup.py build -j4",
+    ["python -m pip install versioneer[toml]",
+     "python setup.py build -j4",
      "PIP_NO_BUILD_ISOLATION=false python -mpip wheel --no-deps --no-index -w {build_cache_dir} {build_dir}"],
 }

asv_bench/benchmarks/frame_ctor.py

@@ -205,21 +205,4 @@ def time_frame_from_arrays_sparse(self):
         )
 
 
-class From3rdParty:
-    # GH#44616
-
-    def setup(self):
-        try:
-            import torch
-        except ImportError:
-            raise NotImplementedError
-
-        row = 700000
-        col = 64
-        self.val_tensor = torch.randn(row, col)
-
-    def time_from_torch(self):
-        DataFrame(self.val_tensor)
-
-
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/frame_methods.py

@@ -454,10 +454,10 @@ def setup(self, axis):
         )
 
     def time_count_level_multi(self, axis):
-        self.df.count(axis=axis, level=1)
+        self.df.count(axis=axis)
 
     def time_count_level_mixed_dtypes_multi(self, axis):
-        self.df_mixed.count(axis=axis, level=1)
+        self.df_mixed.count(axis=axis)
 
 
 class Apply: