From deb7abe72ef683f319e902ad197ff84bde2b4964 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 12:02:59 -0700 Subject: [PATCH 01/11] S3 Management Job Outside Docker --- .github/workflows/update-s3-html.yml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/update-s3-html.yml diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml new file mode 100644 index 000000000..12e428e66 --- /dev/null +++ b/.github/workflows/update-s3-html.yml @@ -0,0 +1,36 @@ +name: Update S3 HTML indices for download.pytorch.org +on: + # TODO: remove before merging + pull_request: + schedule: + # Update the indices every 30 minutes + - cron: "*/30 * * * *" + # Have the ability to trigger this job manually using the API as well + workflow_dispatch: + +jobs: + update-s3: + strategy: + matrix: + prefix: ["whl", "whl/test", "whl/nightly", "whl/lts/1.8"] + fail-fast: false + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + repository: pytorch/builder + timeout: 60 + secrets-env: "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" + script: | + set -ex + + # Create Conda Environment + git config --global --add safe.directory /__w/builder/builder + conda create --quiet -y --prefix run_env python="3.8" + conda activate ./run_env + + # Set Envs + export AWS_ACCESS_KEY_ID = ${SECRET_AWS_ACCESS_KEY_ID} + export AWS_SECRET_ACCESS_KEY = ${SECRET_AWS_SECRET_ACCESS_KEY} + + # Install requirements + pip install -r s3_management/requirements.txt + python s3_management/manage.py --generate-pep503 ${{ matrix.prefix }} From 606730eabcdad6fff1fbe6c20c2feefc36bc5c62 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 13:44:15 -0700 Subject: [PATCH 02/11] job name --- .github/workflows/update-s3-html.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index 12e428e66..6b6468c0b 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: jobs: - update-s3: + update: strategy: matrix: prefix: ["whl", "whl/test", "whl/nightly", "whl/lts/1.8"] From 4dc096b06871eab561f82fc7839f9f2805f799fc Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 13:46:17 -0700 Subject: [PATCH 03/11] remove failfast --- .github/workflows/update-s3-html.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index 6b6468c0b..ffa1be332 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -13,7 +13,6 @@ jobs: strategy: matrix: prefix: ["whl", "whl/test", "whl/nightly", "whl/lts/1.8"] - fail-fast: false uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: repository: pytorch/builder From d68a53a85197e19695a747d17d6f44db8a006178 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 13:47:27 -0700 Subject: [PATCH 04/11] no matrix --- .github/workflows/update-s3-html.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index ffa1be332..f507a67bd 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -10,9 +10,6 @@ on: jobs: update: - strategy: - matrix: - prefix: ["whl", "whl/test", "whl/nightly", "whl/lts/1.8"] uses: pytorch/test-infra/.github/workflows/linux_job.yml@main with: repository: pytorch/builder @@ -32,4 +29,5 @@ jobs: # Install requirements pip install -r s3_management/requirements.txt - python s3_management/manage.py --generate-pep503 ${{ matrix.prefix }} + # python s3_management/manage.py --generate-pep503 ${{ matrix.prefix }} + python s3_management/manage.py --generate-pep503 whl From 718454295cfd9e5b5ff7fe9f62c4003832896ab4 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 13:49:20 -0700 Subject: [PATCH 05/11] inherit secrets --- .github/workflows/update-s3-html.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index f507a67bd..2cc062ea5 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -11,6 +11,7 @@ on: jobs: update: uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + secrets: inherit with: repository: pytorch/builder timeout: 60 From 5c2a4bf981059a45ecf24651ae0ddfa46b75ca16 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 13:52:11 -0700 Subject: [PATCH 06/11] spacing? --- .github/workflows/update-s3-html.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index 2cc062ea5..e0be698e5 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -1,4 +1,5 @@ name: Update S3 HTML indices for download.pytorch.org + on: # TODO: remove before merging pull_request: From 8fac93b12e113023adda18118c2c2b5c3d6af47e Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 14:07:32 -0700 Subject: [PATCH 07/11] random nits --- .github/workflows/update-s3-html.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index e0be698e5..df338ed47 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -1,12 +1,10 @@ name: Update S3 HTML indices for download.pytorch.org on: - # TODO: remove before merging pull_request: schedule: # Update the indices every 30 minutes - cron: "*/30 * * * *" - # Have the ability to trigger this job manually using the API as well workflow_dispatch: jobs: @@ -16,7 +14,6 @@ jobs: with: repository: pytorch/builder timeout: 60 - secrets-env: "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY" script: | set -ex From 21196dd906727699f8544680ffef8d96d014f3ab Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 14:10:41 -0700 Subject: [PATCH 08/11] add back secrets --- .github/workflows/update-s3-html.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index df338ed47..76c2c41df 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -14,6 +14,7 @@ jobs: with: repository: pytorch/builder timeout: 60 + secrets-env: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY script: | set -ex From c93f3de4238e3eacbe5235c94c5ffa17afe96bb0 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 14:12:23 -0700 Subject: [PATCH 09/11] add back matrix --- .github/workflows/update-s3-html.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index 76c2c41df..d7d42c648 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -9,6 +9,10 @@ on: jobs: update: + strategy: + matrix: + prefix: ["whl", "whl/test", "whl/nightly", "whl/lts/1.8"] + fail-fast: False uses: pytorch/test-infra/.github/workflows/linux_job.yml@main secrets: inherit with: @@ -29,5 +33,4 @@ jobs: # Install requirements pip install -r s3_management/requirements.txt - # python s3_management/manage.py --generate-pep503 ${{ matrix.prefix }} - python s3_management/manage.py --generate-pep503 whl + python s3_management/manage.py --generate-pep503 ${{ matrix.prefix }} From bdcde3adb9aa0974ad6b5b6a2b2a50a235310557 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Tue, 12 Sep 2023 14:13:18 -0700 Subject: [PATCH 10/11] export env vars correctlty --- .github/workflows/update-s3-html.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index d7d42c648..9361bd8b0 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -28,8 +28,8 @@ jobs: conda activate ./run_env # Set Envs - export AWS_ACCESS_KEY_ID = ${SECRET_AWS_ACCESS_KEY_ID} - export AWS_SECRET_ACCESS_KEY = ${SECRET_AWS_SECRET_ACCESS_KEY} + export AWS_ACCESS_KEY_ID="${SECRET_AWS_ACCESS_KEY_ID}" + export AWS_SECRET_ACCESS_KEY="${SECRET_AWS_SECRET_ACCESS_KEY}" # Install requirements pip install -r s3_management/requirements.txt From 41ced9d953b41d11e51a9c8a605a13bcf17a9fa9 Mon Sep 17 00:00:00 2001 From: Omkar Salpekar Date: Thu, 14 Sep 2023 14:19:44 -0700 Subject: [PATCH 11/11] Update update-s3-html.yml --- .github/workflows/update-s3-html.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update-s3-html.yml b/.github/workflows/update-s3-html.yml index 9361bd8b0..7c285418e 100644 --- a/.github/workflows/update-s3-html.yml +++ b/.github/workflows/update-s3-html.yml @@ -1,7 +1,6 @@ name: Update S3 HTML indices for download.pytorch.org on: - pull_request: schedule: # Update the indices every 30 minutes - cron: "*/30 * * * *"