From 94b256789ff75956c4eba717ded3dedea75c7e4a Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Wed, 25 May 2022 12:08:05 +0200 Subject: [PATCH 01/11] Add run-tests and setup-pandas actions --- .github/actions/build_pandas/action.yml | 6 ++++ .github/actions/run-tests/action.yml | 41 +++++++++++++++++++++++++ .github/actions/setup-pandas/action.yml | 33 ++++++++++++++++++++ .github/workflows/macos-windows.yml | 40 +++++------------------- setup.py | 2 +- 5 files changed, 88 insertions(+), 34 deletions(-) create mode 100644 .github/actions/run-tests/action.yml create mode 100644 .github/actions/setup-pandas/action.yml diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 5e5a3bdf0f024..5c2060771c5c9 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -8,6 +8,8 @@ runs: run: | conda info conda list + # Make sure we have the correct environment activated + conda info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none' shell: bash -el {0} - name: Build Pandas @@ -18,3 +20,7 @@ runs: env: # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} + + - name: Show dependency versions + run: python -c "import pandas; pandas.show_versions()" + shell: bash -el {0} diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml new file mode 100644 index 0000000000000..add86954cf061 --- /dev/null +++ b/.github/actions/run-tests/action.yml @@ -0,0 +1,41 @@ +name: Run tests and report results +description: >- + Runs the test suite and publishes the results and coverage. + + Pandas is expected to be installed with pip or Conda in a way that a Python + interpreter run from a login bash will be able to import it, eg. + `bash -lc 'python -c "import pandas"'`. +inputs: + check-pyarrow-version: + description: >- + If set, double-check that the PyArrow version used by the Pandas installation + if identical to this string. + required: false +runs: + using: composite + steps: + - name: Check PyArrow version + run: | + # Double check that we have the expected PyArrow + mamba list -f pyarrow | grep ${{ inputs.check-pyarrow-version }} + shell: bash -el {0} + if: ${{ inputs.check-pyarrow-version }} + + - name: Test + run: ci/run_tests.sh + shell: bash -el {0} + + - name: Publish test results + uses: actions/upload-artifact@v2 + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + flags: unittests + name: codecov-pandas + fail_ci_if_error: false + if: failure() diff --git a/.github/actions/setup-pandas/action.yml b/.github/actions/setup-pandas/action.yml new file mode 100644 index 0000000000000..2eb0358bba57f --- /dev/null +++ b/.github/actions/setup-pandas/action.yml @@ -0,0 +1,33 @@ +name: Set up pandas +description: Runs all the setup steps required to have a built pandas ready to use +inputs: + environment-file: + description: Conda environment file to use. + default: environment.yml + pyarrow-version: + description: If set, overrides the PyArrow version in the Conda environment to the given string. + required: false +runs: + using: composite + steps: + - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} + run: | + grep -q ' - pyarrow' ${{ inputs.environment-file }} + sed -i"" -e "s/ - pyarrow/ - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} + cat ${{ inputs.environment-file }} + shell: bash + if: ${{ inputs.pyarrow-version }} + + - name: Pin setuptools (GH#44980) + run: | + echo ' - setuptools <60' >> ${{ inputs.environment-file }} + shell: bash + + - name: Install ${{ inputs.environment-file }} + uses: conda-incubator/setup-miniconda@v2 + with: + environment-file: ${{ inputs.environment-file }} + channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }} + channels: conda-forge + mamba-version: "0.23" + use-mamba: true diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml index 560a421ec74ec..ca9e92b6c0cf9 100644 --- a/.github/workflows/macos-windows.yml +++ b/.github/workflows/macos-windows.yml @@ -43,42 +43,16 @@ jobs: with: fetch-depth: 0 - - name: Install Dependencies - uses: conda-incubator/setup-miniconda@v2.1.1 + - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + uses: ./.github/actions/setup-pandas with: - mamba-version: "*" - channels: conda-forge - activate-environment: pandas-dev - channel-priority: ${{ matrix.os == 'macos-latest' && 'flexible' || 'strict' }} environment-file: ci/deps/${{ matrix.env_file }} - use-only-tar-bz2: true + pyarrow-version: ${{ matrix.os == 'macos-latest' && '6' || '' }} - # ImportError: 2): Library not loaded: @rpath/libssl.1.1.dylib - # Referenced from: /Users/runner/miniconda3/envs/pandas-dev/lib/libthrift.0.13.0.dylib - # Reason: image not found - - name: Upgrade pyarrow on MacOS - run: conda install -n pandas-dev -c conda-forge --no-update-deps pyarrow=6 - if: ${{ matrix.os == 'macos-latest' }} - - - name: Build Pandas + - name: Build pandas uses: ./.github/actions/build_pandas - - name: Test - run: ci/run_tests.sh - - - name: Build Version - run: conda list - - - name: Publish test results - uses: actions/upload-artifact@v3 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 + - name: Run tests + uses: ./.github/actions/run-tests with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false + check-pyarrow-version: ${{ matrix.pyarrow_version }} diff --git a/setup.py b/setup.py index 0418fe67c2f76..ced8b8dbc96c6 100755 --- a/setup.py +++ b/setup.py @@ -333,7 +333,7 @@ def run(self): extra_compile_args.append("/Z7") extra_link_args.append("/DEBUG") else: - # PANDAS_CI=1 is set by ci/setup_env.sh + # PANDAS_CI=1 is set in CI if os.environ.get("PANDAS_CI", "0") == "1": extra_compile_args.append("-Werror") if debugging_symbols_requested: From 4d22885504726173f576fa6158551e7ee1bfeb08 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 28 May 2022 00:23:47 +0200 Subject: [PATCH 02/11] Undo run-tests --- .github/actions/run-tests/action.yml | 41 ---------------------------- .github/workflows/macos-windows.yml | 20 +++++++++++--- 2 files changed, 16 insertions(+), 45 deletions(-) delete mode 100644 .github/actions/run-tests/action.yml diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml deleted file mode 100644 index add86954cf061..0000000000000 --- a/.github/actions/run-tests/action.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Run tests and report results -description: >- - Runs the test suite and publishes the results and coverage. - - Pandas is expected to be installed with pip or Conda in a way that a Python - interpreter run from a login bash will be able to import it, eg. - `bash -lc 'python -c "import pandas"'`. -inputs: - check-pyarrow-version: - description: >- - If set, double-check that the PyArrow version used by the Pandas installation - if identical to this string. - required: false -runs: - using: composite - steps: - - name: Check PyArrow version - run: | - # Double check that we have the expected PyArrow - mamba list -f pyarrow | grep ${{ inputs.check-pyarrow-version }} - shell: bash -el {0} - if: ${{ inputs.check-pyarrow-version }} - - - name: Test - run: ci/run_tests.sh - shell: bash -el {0} - - - name: Publish test results - uses: actions/upload-artifact@v2 - with: - name: Test results - path: test-data.xml - if: failure() - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: codecov-pandas - fail_ci_if_error: false - if: failure() diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml index ca9e92b6c0cf9..eae0297220ce2 100644 --- a/.github/workflows/macos-windows.yml +++ b/.github/workflows/macos-windows.yml @@ -49,10 +49,22 @@ jobs: environment-file: ci/deps/${{ matrix.env_file }} pyarrow-version: ${{ matrix.os == 'macos-latest' && '6' || '' }} - - name: Build pandas + - name: Build Pandas uses: ./.github/actions/build_pandas - - name: Run tests - uses: ./.github/actions/run-tests + - name: Test + run: ci/run_tests.sh + + - name: Publish test results + uses: actions/upload-artifact@v3 + with: + name: Test results + path: test-data.xml + if: failure() + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 with: - check-pyarrow-version: ${{ matrix.pyarrow_version }} + flags: unittests + name: codecov-pandas + fail_ci_if_error: false From 7a01e03710c75b8cb88fb54ab3b5c2b8dfe24ba7 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sat, 28 May 2022 01:19:54 +0200 Subject: [PATCH 03/11] Use setup-pandas in docbuild --- .circleci/config.yml | 2 +- {ci => .circleci}/setup_env.sh | 0 .github/actions/setup/action.yml | 12 ------------ .github/workflows/docbuild-and-upload.yml | 2 +- 4 files changed, 2 insertions(+), 14 deletions(-) rename {ci => .circleci}/setup_env.sh (100%) delete mode 100644 .github/actions/setup/action.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 612552f4eac59..0d9e3ade08846 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: PANDAS_CI: "1" steps: - checkout - - run: ci/setup_env.sh + - run: .circleci/setup_env.sh - run: PATH=$HOME/miniconda3/envs/pandas-dev/bin:$HOME/miniconda3/condabin:$PATH ci/run_tests.sh workflows: diff --git a/ci/setup_env.sh b/.circleci/setup_env.sh similarity index 100% rename from ci/setup_env.sh rename to .circleci/setup_env.sh diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml deleted file mode 100644 index c357f149f2c7f..0000000000000 --- a/.github/actions/setup/action.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Set up pandas -description: Runs all the setup steps required to have a built pandas ready to use -runs: - using: composite - steps: - - name: Setting conda path - run: echo "${HOME}/miniconda3/bin" >> $GITHUB_PATH - shell: bash -el {0} - - - name: Setup environment and build pandas - run: ci/setup_env.sh - shell: bash -el {0} diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 8c2c86dc693a9..8c545e9109726 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 0 - name: Set up pandas - uses: ./.github/actions/setup + uses: ./.github/actions/setup-pandas - name: Build website run: | From ac55a000b60b76ef79f20ac5ee4dfb640a6af797 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sun, 29 May 2022 19:09:46 +0200 Subject: [PATCH 04/11] Update docbuild-and-upload.yml --- .github/workflows/docbuild-and-upload.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 8c545e9109726..980a60380f68a 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -23,6 +23,10 @@ jobs: # https://github.community/t/concurrecy-not-work-for-push/183068/7 group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs cancel-in-progress: true + + defaults: + run: + shell: bash -el {0} steps: - name: Checkout @@ -34,14 +38,10 @@ jobs: uses: ./.github/actions/setup-pandas - name: Build website - run: | - source activate pandas-dev - python web/pandas_web.py web/pandas --target-path=web/build + run: python web/pandas_web.py web/pandas --target-path=web/build - name: Build documentation - run: | - source activate pandas-dev - doc/make.py --warnings-are-errors + run: doc/make.py --warnings-are-errors - name: Install ssh key run: | @@ -49,18 +49,18 @@ jobs: echo "${{ secrets.server_ssh_key }}" > ~/.ssh/id_rsa chmod 600 ~/.ssh/id_rsa echo "${{ secrets.server_ip }} ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE1Kkopomm7FHG5enATf7SgnpICZ4W2bw+Ho+afqin+w7sMcrsa0je7sbztFAV8YchDkiBKnWTG4cRT+KZgZCaY=" > ~/.ssh/known_hosts - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' - name: Copy cheatsheets into site directory run: cp doc/cheatsheet/Pandas_Cheat_Sheet* web/build/ - name: Upload web run: rsync -az --delete --exclude='pandas-docs' --exclude='docs' web/build/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' - name: Upload dev docs run: rsync -az --delete doc/build/html/ docs@${{ secrets.server_ip }}:/usr/share/nginx/pandas/pandas-docs/dev - if: ${{github.event_name == 'push' && github.ref == 'refs/heads/main'}} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' - name: Move docs into site directory run: mv doc/build/html web/build/docs From 5e7d8029c311381cae74730f6bceeb4b7fbf9c4f Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sun, 29 May 2022 20:00:45 +0200 Subject: [PATCH 05/11] Update macos-windows.yml --- .github/workflows/macos-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml index eae0297220ce2..385c8d3d6108c 100644 --- a/.github/workflows/macos-windows.yml +++ b/.github/workflows/macos-windows.yml @@ -44,7 +44,7 @@ jobs: fetch-depth: 0 - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) - uses: ./.github/actions/setup-pandas + uses: ./.github/actions/setup-conda with: environment-file: ci/deps/${{ matrix.env_file }} pyarrow-version: ${{ matrix.os == 'macos-latest' && '6' || '' }} From b22364f8fc509f1a253fa1d090a3cd6979e066eb Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sun, 29 May 2022 20:01:44 +0200 Subject: [PATCH 06/11] Update docbuild-and-upload.yml --- .github/workflows/docbuild-and-upload.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docbuild-and-upload.yml b/.github/workflows/docbuild-and-upload.yml index 980a60380f68a..5ffd4135802bd 100644 --- a/.github/workflows/docbuild-and-upload.yml +++ b/.github/workflows/docbuild-and-upload.yml @@ -23,7 +23,7 @@ jobs: # https://github.community/t/concurrecy-not-work-for-push/183068/7 group: ${{ github.event_name == 'push' && github.run_number || github.ref }}-web-docs cancel-in-progress: true - + defaults: run: shell: bash -el {0} @@ -34,8 +34,11 @@ jobs: with: fetch-depth: 0 - - name: Set up pandas - uses: ./.github/actions/setup-pandas + - name: Set up Conda + uses: ./.github/actions/setup-conda + + - name: Build Pandas + uses: ./.github/actions/build_pandas - name: Build website run: python web/pandas_web.py web/pandas --target-path=web/build From 867f1e73be240163e145b1ece800a776354e960c Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sun, 29 May 2022 20:02:39 +0200 Subject: [PATCH 07/11] Update action.yml --- .github/actions/setup-pandas/action.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/actions/setup-pandas/action.yml b/.github/actions/setup-pandas/action.yml index 2eb0358bba57f..1c947ff244fb9 100644 --- a/.github/actions/setup-pandas/action.yml +++ b/.github/actions/setup-pandas/action.yml @@ -1,5 +1,4 @@ -name: Set up pandas -description: Runs all the setup steps required to have a built pandas ready to use +name: Set up Conda environment inputs: environment-file: description: Conda environment file to use. @@ -18,11 +17,6 @@ runs: shell: bash if: ${{ inputs.pyarrow-version }} - - name: Pin setuptools (GH#44980) - run: | - echo ' - setuptools <60' >> ${{ inputs.environment-file }} - shell: bash - - name: Install ${{ inputs.environment-file }} uses: conda-incubator/setup-miniconda@v2 with: From e017dcc4d39e9cf09b198778aa8b30148ff37a61 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sun, 29 May 2022 20:03:28 +0200 Subject: [PATCH 08/11] Create action.yml --- .github/actions/setup-conda/action.yml | 27 ++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/actions/setup-conda/action.yml diff --git a/.github/actions/setup-conda/action.yml b/.github/actions/setup-conda/action.yml new file mode 100644 index 0000000000000..1c947ff244fb9 --- /dev/null +++ b/.github/actions/setup-conda/action.yml @@ -0,0 +1,27 @@ +name: Set up Conda environment +inputs: + environment-file: + description: Conda environment file to use. + default: environment.yml + pyarrow-version: + description: If set, overrides the PyArrow version in the Conda environment to the given string. + required: false +runs: + using: composite + steps: + - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} + run: | + grep -q ' - pyarrow' ${{ inputs.environment-file }} + sed -i"" -e "s/ - pyarrow/ - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} + cat ${{ inputs.environment-file }} + shell: bash + if: ${{ inputs.pyarrow-version }} + + - name: Install ${{ inputs.environment-file }} + uses: conda-incubator/setup-miniconda@v2 + with: + environment-file: ${{ inputs.environment-file }} + channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }} + channels: conda-forge + mamba-version: "0.23" + use-mamba: true From afbb61034882853df122de869e71118447b0dd51 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Sun, 29 May 2022 20:03:42 +0200 Subject: [PATCH 09/11] Delete .github/actions/setup-pandas directory --- .github/actions/setup-pandas/action.yml | 27 ------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .github/actions/setup-pandas/action.yml diff --git a/.github/actions/setup-pandas/action.yml b/.github/actions/setup-pandas/action.yml deleted file mode 100644 index 1c947ff244fb9..0000000000000 --- a/.github/actions/setup-pandas/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Set up Conda environment -inputs: - environment-file: - description: Conda environment file to use. - default: environment.yml - pyarrow-version: - description: If set, overrides the PyArrow version in the Conda environment to the given string. - required: false -runs: - using: composite - steps: - - name: Set Arrow version in ${{ inputs.environment-file }} to ${{ inputs.pyarrow-version }} - run: | - grep -q ' - pyarrow' ${{ inputs.environment-file }} - sed -i"" -e "s/ - pyarrow/ - pyarrow=${{ inputs.pyarrow-version }}/" ${{ inputs.environment-file }} - cat ${{ inputs.environment-file }} - shell: bash - if: ${{ inputs.pyarrow-version }} - - - name: Install ${{ inputs.environment-file }} - uses: conda-incubator/setup-miniconda@v2 - with: - environment-file: ${{ inputs.environment-file }} - channel-priority: ${{ runner.os == 'macOS' && 'flexible' || 'strict' }} - channels: conda-forge - mamba-version: "0.23" - use-mamba: true From 8106b10a67003283108af0c0415626f638925e36 Mon Sep 17 00:00:00 2001 From: Jonas Haag Date: Tue, 31 May 2022 09:22:27 +0200 Subject: [PATCH 10/11] Review feedback --- .github/actions/build_pandas/action.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/actions/build_pandas/action.yml b/.github/actions/build_pandas/action.yml index 5c2060771c5c9..5e5a3bdf0f024 100644 --- a/.github/actions/build_pandas/action.yml +++ b/.github/actions/build_pandas/action.yml @@ -8,8 +8,6 @@ runs: run: | conda info conda list - # Make sure we have the correct environment activated - conda info | grep -Ei 'environment.+:' | grep -qEiv 'environment.+:.+none' shell: bash -el {0} - name: Build Pandas @@ -20,7 +18,3 @@ runs: env: # Cannot use parallel compilation on Windows, see https://github.com/pandas-dev/pandas/issues/30873 N_JOBS: ${{ runner.os == 'Windows' && 1 || 2 }} - - - name: Show dependency versions - run: python -c "import pandas; pandas.show_versions()" - shell: bash -el {0} From 4b3f8c473ad27abe353a63be1ebfd32449fcb0bd Mon Sep 17 00:00:00 2001 From: Matthew Roeschke Date: Fri, 3 Jun 2022 12:26:18 -0700 Subject: [PATCH 11/11] Update .github/workflows/macos-windows.yml --- .github/workflows/macos-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-windows.yml b/.github/workflows/macos-windows.yml index 385c8d3d6108c..26e6c8699ca64 100644 --- a/.github/workflows/macos-windows.yml +++ b/.github/workflows/macos-windows.yml @@ -43,7 +43,7 @@ jobs: with: fetch-depth: 0 - - name: Set up Conda (${{ matrix.env_file }}, Arrow ${{ matrix.pyarrow_version}}) + - name: Set up Conda uses: ./.github/actions/setup-conda with: environment-file: ci/deps/${{ matrix.env_file }}