From e7fa1c3d2ee27e12e4b5e24ea9909ee2b00a3d7a Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 16 May 2023 10:50:48 -0700 Subject: [PATCH 1/6] Remove circleci doc build job --- .circleci/config.yml | 128 ++++-------------------- .circleci/config.yml.in | 217 +--------------------------------------- .circleci/regenerate.py | 140 +------------------------- 3 files changed, 21 insertions(+), 464 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 500453ce87..81bfd9a4bd 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,6 +133,7 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 +<<<<<<< HEAD binary_linux_wheel: <<: *binary_common docker: @@ -229,6 +230,8 @@ jobs: aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read done +======= +>>>>>>> 4ea6a71df (Remove circleci doc build job) smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 @@ -249,55 +252,6 @@ jobs: docker push ${image_name}:${CIRCLE_WORKFLOW_ID} docker push ${image_name}:latest - smoke_test_windows_conda: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - load_conda_channel_flags - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda env remove -n python${PYTHON_VERSION} || true - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-"${UPLOAD_CHANNEL}" pytorch - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c ~/workspace/conda-bld torchtext - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchtext" - - smoke_test_windows_pip: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda env remove -n python${PYTHON_VERSION} || true - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchtext" stylecheck: <<: *binary_common @@ -313,73 +267,13 @@ jobs: - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh - build_docs: - <<: *binary_common - docker: - - image: continuumio/miniconda3 - resource_class: medium - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - checkout - - run: - name: install binaries - command: | - set -x - conda install -y make python=${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cpu/torch_${UPLOAD_CHANNEL}.html" - - run: - name: Build docs - command: | - set -x - pushd docs - pip install -r requirements.txt - BUILD_GALLERY=1 make 'SPHINXOPTS=-W' html - popd - - persist_to_workspace: - root: ./ - paths: - - "*" - - store_artifacts: - path: ./docs/build/html - destination: docs - - upload_docs: - <<: *binary_common - docker: - - image: continuumio/miniconda3 - resource_class: medium - steps: - - attach_workspace: - at: ~/workspace - - run: - name: Generate netrc - command: | - # set credentials for https pushing - # requires the org-member context - cat > ~/.netrc \<>>>>>> 4ea6a71df (Remove circleci doc build job) docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 781abdc81e..a491dfcd37 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -133,102 +133,6 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 - binary_linux_wheel: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - run: packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_windows_wheel: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - run: - name: build - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - bash packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_windows_conda: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: build - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda install ${CONDA_CHANNEL_FLAGS} -yq conda-build "conda-package-handling!=1.5.0" - bash packaging/build_conda.sh - rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2 - - store_artifacts: - path: C:/tools/miniconda3/conda-bld/win-64 - - persist_to_workspace: - root: C:/tools/miniconda3 - paths: - - "conda-bld/*" - - # Requires org-member context - binary_conda_upload: - docker: - - image: continuumio/miniconda - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - command: | - # Prevent credential from leaking - conda install -yq anaconda-client - set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force - - # Requires org-member context - binary_wheel_upload: - docker: - - image: cimg/python:3.8 - steps: - - attach_workspace: - at: ~/workspace - - checkout - - designate_upload_channel - - run: - command: | - pip install --user awscli - export PATH="$HOME/.local/bin:$PATH" - # Prevent credential from leaking - set +x - export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" - set -x - for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read - done - smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 @@ -249,55 +153,6 @@ jobs: docker push ${image_name}:${CIRCLE_WORKFLOW_ID} docker push ${image_name}:latest - smoke_test_windows_conda: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - load_conda_channel_flags - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda env remove -n python${PYTHON_VERSION} || true - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c pytorch-"${UPLOAD_CHANNEL}" pytorch - conda install -v -y ${CONDA_CHANNEL_FLAGS} -c ~/workspace/conda-bld torchtext - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchtext" - - smoke_test_windows_pip: - <<: *binary_common - executor: - name: windows-cpu - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - name: install binaries - command: | - set -x - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda env remove -n python${PYTHON_VERSION} || true - conda create -yn python${PYTHON_VERSION} python=${PYTHON_VERSION} - conda activate python${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/torch_${UPLOAD_CHANNEL}.html" - - run: - name: smoke test - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate python${PYTHON_VERSION} - python -c "import torchtext" stylecheck: <<: *binary_common @@ -313,83 +168,15 @@ jobs: - run: name: Run style check command: .circleci/unittest/linux/scripts/run_style_checks.sh - build_docs: - <<: *binary_common - docker: - - image: continuumio/miniconda3 - resource_class: medium - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - checkout - - run: - name: install binaries - command: | - set -x - conda install -y make python=${PYTHON_VERSION} - pip install $(ls ~/workspace/torchtext*.whl) --pre -f "https://download.pytorch.org/whl/${UPLOAD_CHANNEL}/cpu/torch_${UPLOAD_CHANNEL}.html" - - run: - name: Build docs - command: | - set -x - pushd docs - pip install -r requirements.txt - BUILD_GALLERY=1 make 'SPHINXOPTS=-W' html - popd - - persist_to_workspace: - root: ./ - paths: - - "*" - - store_artifacts: - path: ./docs/build/html - destination: docs - - upload_docs: - <<: *binary_common - docker: - - image: continuumio/miniconda3 - resource_class: medium - steps: - - attach_workspace: - at: ~/workspace - - run: - name: Generate netrc - command: | - # set credentials for https pushing - # requires the org-member context - cat > ~/.netrc \< Date: Tue, 16 May 2023 11:15:46 -0700 Subject: [PATCH 2/6] Rebase --- .circleci/config.yml | 156 ---------------------------------------- .circleci/config.yml.in | 5 -- 2 files changed, 161 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 81bfd9a4bd..a491dfcd37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -133,105 +133,6 @@ jobs: name: Code format not compliant with the rules! Run '$ python run-clang-format.py' to fix this. command: exit 1 -<<<<<<< HEAD - binary_linux_wheel: - <<: *binary_common - docker: - - image: "pytorch/manylinux-cuda102" - resource_class: 2xlarge+ - steps: - - checkout - - designate_upload_channel - - run: packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_windows_wheel: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - run: - name: build - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - bash packaging/build_wheel.sh - - store_artifacts: - path: dist - - persist_to_workspace: - root: dist - paths: - - "*" - - binary_windows_conda: - <<: *binary_common - executor: - name: windows-cpu - steps: - - checkout - - designate_upload_channel - - load_conda_channel_flags - - run: - name: build - command: | - eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')" - conda activate base - conda install ${CONDA_CHANNEL_FLAGS} -yq conda-build "conda-package-handling!=1.5.0" - bash packaging/build_conda.sh - rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2 - - store_artifacts: - path: C:/tools/miniconda3/conda-bld/win-64 - - persist_to_workspace: - root: C:/tools/miniconda3 - paths: - - "conda-bld/*" - - # Requires org-member context - binary_conda_upload: - docker: - - image: continuumio/miniconda - steps: - - attach_workspace: - at: ~/workspace - - designate_upload_channel - - run: - command: | - # Prevent credential from leaking - conda install -yq anaconda-client - set -x - anaconda -t "${CONDA_PYTORCHBOT_TOKEN}" upload ~/workspace/conda-bld/*/*.tar.bz2 -u "pytorch-${UPLOAD_CHANNEL}" --label main --no-progress --force - - # Requires org-member context - binary_wheel_upload: - docker: - - image: cimg/python:3.8 - steps: - - attach_workspace: - at: ~/workspace - - checkout - - designate_upload_channel - - run: - command: | - pip install --user awscli - export PATH="$HOME/.local/bin:$PATH" - # Prevent credential from leaking - set +x - export AWS_ACCESS_KEY_ID="${PYTORCH_BINARY_AWS_ACCESS_KEY_ID}" - export AWS_SECRET_ACCESS_KEY="${PYTORCH_BINARY_AWS_SECRET_ACCESS_KEY}" - set -x - for pkg in ~/workspace/*.whl; do - aws s3 cp "$pkg" "s3://pytorch/whl/${UPLOAD_CHANNEL}/" --acl public-read - done - -======= ->>>>>>> 4ea6a71df (Remove circleci doc build job) smoke_test_docker_image_build: machine: image: ubuntu-1604:201903-01 @@ -273,66 +174,9 @@ workflows: - circleci_consistency - lint_python_and_config - lint_c -<<<<<<< HEAD - build: - jobs: - - binary_linux_wheel: - filters: - branches: - only: /.*/ - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: binary_linux_wheel_py3.8 - python_version: '3.8' - - build_docs: - filters: - branches: - only: /.*/ - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: build_docs - python_version: '3.8' - requires: - - binary_linux_wheel_py3.8 - - upload_docs: - context: org-member - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: upload_docs - python_version: '3.8' - requires: - - build_docs - nightly: - jobs: - - circleci_consistency: - filters: - branches: - only: nightly - - binary_linux_wheel: - filters: - branches: - only: nightly - tags: - only: /v[0-9]+(\.[0-9]+)*-rc[0-9]+/ - name: nightly_binary_linux_wheel_py3.8 - python_version: '3.8' -======= unittest: jobs: - circleci_consistency - - unittest_windows: - name: unittest_windows_py3.8 - python_version: '3.8' - - unittest_windows: - name: unittest_windows_py3.9 - python_version: '3.9' - - unittest_windows: - name: unittest_windows_py3.10 - python_version: '3.10' ->>>>>>> 4ea6a71df (Remove circleci doc build job) docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index a491dfcd37..6b8442b6a3 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -64,11 +64,6 @@ binary_common: &binary_common CU_VERSION: cpu MACOSX_DEPLOYMENT_TARGET: 10.9 -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: pytorch/torchtext_smoke_base:smoke_test-20220427 - jobs: circleci_consistency: docker: From a78396b40415225c803162ac322dc277a7579b3e Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 16 May 2023 11:17:53 -0700 Subject: [PATCH 3/6] Remove unittests --- .circleci/config.yml | 8 -------- .circleci/config.yml.in | 3 --- 2 files changed, 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a491dfcd37..a426f613c5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -64,11 +64,6 @@ binary_common: &binary_common CU_VERSION: cpu MACOSX_DEPLOYMENT_TARGET: 10.9 -smoke_test_common: &smoke_test_common - <<: *binary_common - docker: - - image: pytorch/torchtext_smoke_base:smoke_test-20220427 - jobs: circleci_consistency: docker: @@ -174,9 +169,6 @@ workflows: - circleci_consistency - lint_python_and_config - lint_c - unittest: - jobs: - - circleci_consistency docker_build: triggers: - schedule: diff --git a/.circleci/config.yml.in b/.circleci/config.yml.in index 6b8442b6a3..a426f613c5 100644 --- a/.circleci/config.yml.in +++ b/.circleci/config.yml.in @@ -169,9 +169,6 @@ workflows: - circleci_consistency - lint_python_and_config - lint_c - unittest: - jobs: - - circleci_consistency docker_build: triggers: - schedule: From 486dda31efc31bdb06a43504209fdfbe92c13c31 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 16 May 2023 11:47:33 -0700 Subject: [PATCH 4/6] Fix ufmt --- .circleci/regenerate.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index 40b6ad9f3c..e06f9ba07d 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -21,7 +21,6 @@ from jinja2 import select_autoescape - if __name__ == "__main__": d = os.path.dirname(__file__) env = jinja2.Environment( @@ -31,7 +30,5 @@ ) with open(os.path.join(d, "config.yml"), "w") as f: - f.write( - env.get_template("config.yml.in").render() - ) + f.write(env.get_template("config.yml.in").render()) f.write("\n") From fb1731afe6091752c810a454d1f95de27b3b2ec4 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 16 May 2023 11:48:40 -0700 Subject: [PATCH 5/6] flake8 --- .circleci/regenerate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/.circleci/regenerate.py b/.circleci/regenerate.py index e06f9ba07d..f3962ee084 100755 --- a/.circleci/regenerate.py +++ b/.circleci/regenerate.py @@ -17,7 +17,6 @@ import os.path import jinja2 -import yaml from jinja2 import select_autoescape From 38c1ef1b4e1ba6737ffe7c0c8e27efb4c597b007 Mon Sep 17 00:00:00 2001 From: atalman Date: Tue, 16 May 2023 12:01:29 -0700 Subject: [PATCH 6/6] Format prettier --- .github/workflows/build-conda-windows.yml | 4 +-- .github/workflows/build-docs.yml | 39 +++++++++++----------- .github/workflows/build-wheels-windows.yml | 4 +-- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build-conda-windows.yml b/.github/workflows/build-conda-windows.yml index 38bfb6307f..db1037574a 100644 --- a/.github/workflows/build-conda-windows.yml +++ b/.github/workflows/build-conda-windows.yml @@ -30,7 +30,7 @@ jobs: include: - repository: pytorch/text pre-script: packaging/install_torchdata.sh - post-script: '' + post-script: "" conda-package-directory: packaging/torchtext smoke-test-script: test/smoke_tests/smoke_tests.py package-name: torchtext @@ -39,7 +39,7 @@ jobs: with: conda-package-directory: ${{ matrix.conda-package-directory }} repository: ${{ matrix.repository }} - ref: '' + ref: "" test-infra-repository: pytorch/test-infra test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }} diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 3ea03dd7a1..c7a82d0aca 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -68,7 +68,8 @@ jobs: mv docs/build/html /artifacts/ commit: - if: ${{ (github.repository == 'pytorch/text') && ((github.event_name == 'push') && (github.ref_name == 'nightly')) }} + if: + ${{ (github.repository == 'pytorch/text') && ((github.event_name == 'push') && (github.ref_name == 'nightly')) }} permissions: # Required for `git push` # Note: @@ -79,23 +80,23 @@ jobs: runs-on: ubuntu-latest needs: build steps: - - uses: actions/checkout@v3 - with: - ref: gh-pages - - uses: actions/download-artifact@v3 - with: - name: docs - - name: Update nightly doc - run: | - set -x + - uses: actions/checkout@v3 + with: + ref: gh-pages + - uses: actions/download-artifact@v3 + with: + name: docs + - name: Update nightly doc + run: | + set -x - # TODO: add tag-based process (need to handle the main directory name) - rm -rf main - mv html main + # TODO: add tag-based process (need to handle the main directory name) + rm -rf main + mv html main - # Update the main doc - git add --all main || true - git config user.name "pytorchbot" - git config user.email "soumith+bot@pytorch.org" - git commit -m "auto-generating sphinx docs" || true - git push + # Update the main doc + git add --all main || true + git config user.name "pytorchbot" + git config user.email "soumith+bot@pytorch.org" + git commit -m "auto-generating sphinx docs" || true + git push diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml index 7cb5cda2bf..9a43b6b4b9 100644 --- a/.github/workflows/build-wheels-windows.yml +++ b/.github/workflows/build-wheels-windows.yml @@ -31,14 +31,14 @@ jobs: - repository: pytorch/text pre-script: packaging/install_torchdata.sh env-script: packaging/vc_env_helper.bat - post-script: '' + post-script: "" smoke-test-script: test/smoke_tests/smoke_tests.py package-name: torchtext name: ${{ matrix.repository }} uses: pytorch/test-infra/.github/workflows/build_wheels_windows.yml@main with: repository: ${{ matrix.repository }} - ref: '' + ref: "" test-infra-repository: pytorch/test-infra test-infra-ref: main build-matrix: ${{ needs.generate-matrix.outputs.matrix }}