Backport PR #52211 on branch 2.0.x (CI: Test pyarrow nightly instead of intermediate versions) (#52313)

Backport PR #52211: CI: Test pyarrow nightly instead of intermediate versions

Author: mroeschke

.github/actions/setup-conda/action.yml

@@ -9,20 +9,9 @@ inputs:
   extra-specs:
     description: Extra packages to install
     required: false
-  pyarrow-version:
-    description: If set, overrides the PyArrow version in the Conda environment to the given string.
-    required: false
 runs:
   using: composite
   steps:
-    - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }}
-      run: |
-        grep -q '  - pyarrow' ${{ inputs.environment-file }}
-        sed -i"" -e "s/  - pyarrow/  - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }}
-        cat ${{ inputs.environment-file }}
-      shell: bash
-      if: ${{ inputs.pyarrow-version }}
-
     - name: Install ${{ inputs.environment-file }}
       uses: mamba-org/provision-with-micromamba@v12
       with:

.github/workflows/macos-windows.yml

@@ -51,7 +51,6 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ci/deps/${{ matrix.env_file }}
-        pyarrow-version: ${{ matrix.os == 'macos-latest' && '9' || '' }}
 
     - name: Build Pandas
       uses: ./.github/actions/build_pandas

.github/workflows/ubuntu.yml

@@ -27,7 +27,6 @@ jobs:
         env_file: [actions-38.yaml, actions-39.yaml, actions-310.yaml, actions-311.yaml]
         # Prevent the include jobs from overriding other jobs
         pattern: [""]
-        pyarrow_version: ["8", "9", "10"]
         include:
           - name: "Downstream Compat"
             env_file: actions-38-downstream_compat.yaml
@@ -75,21 +74,11 @@ jobs:
             # TODO(cython3): Re-enable once next-beta(after beta 1) comes out
             # There are some warnings failing the build with -werror
             pandas_ci: "0"
-        exclude:
-          - env_file: actions-38.yaml
-            pyarrow_version: "8"
-          - env_file: actions-38.yaml
-            pyarrow_version: "9"
-          - env_file: actions-39.yaml
-            pyarrow_version: "8"
-          - env_file: actions-39.yaml
-            pyarrow_version: "9"
-          - env_file: actions-310.yaml
-            pyarrow_version: "8"
-          - env_file: actions-310.yaml
-            pyarrow_version: "9"
+          - name: "Pyarrow Nightly"
+            env_file: actions-311-pyarrownightly.yaml
+            pattern: "not slow and not network and not single_cpu"
       fail-fast: false
-    name: ${{ matrix.name || format('{0} pyarrow={1} {2}', matrix.env_file, matrix.pyarrow_version, matrix.pattern) }}
+    name: ${{ matrix.name || matrix.env_file }}
     env:
       ENV_FILE: ci/deps/${{ matrix.env_file }}
       PATTERN: ${{ matrix.pattern }}
@@ -107,7 +96,7 @@ jobs:
       COVERAGE: ${{ !contains(matrix.env_file, 'pypy') }}
     concurrency:
       # https://github.community/t/concurrecy-not-work-for-push/183068/7
-      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.pyarrow_version || '' }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
+      group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-${{ matrix.env_file }}-${{ matrix.pattern }}-${{ matrix.extra_apt || '' }}-${{ matrix.pandas_data_manager || '' }}
       cancel-in-progress: true
 
     services:
@@ -166,7 +155,6 @@ jobs:
       uses: ./.github/actions/setup-conda
       with:
         environment-file: ${{ env.ENV_FILE }}
-        pyarrow-version: ${{ matrix.pyarrow_version }}
 
     - name: Build Pandas
       id: build

ci/deps/actions-310.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2
   - pymysql
   - pytables
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat
   - python-snappy
   - pyxlsb

ci/deps/actions-311-pyarrownightly.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - conda-forge
+dependencies:
+  - python=3.11
+
+  # build dependencies
+  - versioneer[toml]
+  - cython>=0.29.33
+
+  # test dependencies
+  - pytest>=7.0.0
+  - pytest-cov
+  - pytest-xdist>=2.2.0
+  - hypothesis>=6.34.2
+  - pytest-asyncio>=0.17.0
+
+  # required dependencies
+  - python-dateutil
+  - numpy
+  - pytz
+  - pip
+
+  - pip:
+    - "tzdata>=2022.1"
+    - "--extra-index-url https://pypi.fury.io/arrow-nightlies/"
+    - "--prefer-binary"
+    - "--pre"
+    - "pyarrow"

ci/deps/actions-311.yaml

@@ -41,7 +41,7 @@ dependencies:
   - psycopg2
   - pymysql
   # - pytables>=3.8.0  # first version that supports 3.11
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat
   - python-snappy
   - pyxlsb

ci/deps/actions-38-downstream_compat.yaml

@@ -39,7 +39,7 @@ dependencies:
   - openpyxl<3.1.1
   - odfpy
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   - pyreadstat
   - pytables

ci/deps/actions-38.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   - pyreadstat
   - pytables

ci/deps/actions-39.yaml

@@ -40,7 +40,7 @@ dependencies:
   - pandas-gbq
   - psycopg2
   - pymysql
-  - pyarrow
+  - pyarrow>=7.0.0
   - pyreadstat
   - pytables
   - python-snappy

ci/deps/circle-38-arm64.yaml

@@ -39,7 +39,7 @@ dependencies:
   - odfpy
   - pandas-gbq
   - psycopg2
-  - pyarrow
+  - pyarrow>=7.0.0
   - pymysql
   # Not provided on ARM
   #- pyreadstat