diff --git a/.circleci/scripts/binary_checkout.sh b/.circleci/scripts/binary_checkout.sh deleted file mode 100755 index b634f5c9a..000000000 --- a/.circleci/scripts/binary_checkout.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Derived from https://github.com/pytorch/pytorch/blob/2c7df1360aa17d4a6d6726998eede3671bcb36ee/.circleci/scripts/binary_populate_env.sh - -set -eux -o pipefail - -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - - -# This step runs on multiple executors with different envfile locations -if [[ "$OSTYPE" == "msys" ]]; then - # windows executor (builds and tests) - rm -rf /c/w - ln -s "${HOME}" /c/w - WORK_DIR="/c/w" -elif [[ -d "/home/circleci/project" ]]; then - # machine executor (binary tests) - WORK_DIR="${HOME}/project" -else - # macos executor (builds and tests) - # docker executor (binary builds) - WORK_DIR="${HOME}" -fi - -if [[ "$OSTYPE" == "msys" ]]; then - # We need to make the paths as short as possible on Windows - PYTORCH_ROOT="$WORK_DIR/p" - BUILDER_ROOT="$WORK_DIR/b" -else - PYTORCH_ROOT="$WORK_DIR/pytorch" - BUILDER_ROOT="$WORK_DIR/builder" -fi - -# Persist these variables for the subsequent steps -echo "export WORK_DIR=${WORK_DIR}" >> ${BASH_ENV} -echo "export PYTORCH_ROOT=${PYTORCH_ROOT}" >> ${BASH_ENV} -echo "export BUILDER_ROOT=${BUILDER_ROOT}" >> ${BASH_ENV} - -# Clone the Pytorch branch -retry git clone --depth 1 https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT" -# Removed checking out pytorch/pytorch using CIRCLE_PR_NUMBER and CIRCLE_SHA1 as -# those environment variables are tied to the host repo where the build is being -# triggered. -retry git submodule update --init --recursive -pushd "$PYTORCH_ROOT" -echo "Using Pytorch from " -git --no-pager log --max-count 1 -popd - -# Clone the Builder master repo -retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT" -pushd "$BUILDER_ROOT" -if [[ -n "${CIRCLE_SHA1:-}" ]]; then - # Check out a specific commit (typically the latest) from pytorch/builder - git reset --hard "${CIRCLE_SHA1}" - git checkout -q -B main -fi -echo "Using builder from " -git --no-pager log --max-count 1 -popd diff --git a/.circleci/scripts/binary_populate_env.sh b/.circleci/scripts/binary_populate_env.sh index f9a542b53..32f6cc384 100755 --- a/.circleci/scripts/binary_populate_env.sh +++ b/.circleci/scripts/binary_populate_env.sh @@ -144,7 +144,6 @@ export BUILD_JNI=$BUILD_JNI export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER" export DOCKER_IMAGE="$DOCKER_IMAGE" -# Remove WORKD_DIR, PYTORCH_ROOT, BUILDER_ROOT defined & persisted in binary_checkout.sh export MAC_PACKAGE_WORK_DIR="$WORK_DIR" export MINICONDA_ROOT="$WORK_DIR/miniconda" export PYTORCH_FINAL_PACKAGE_DIR="$WORK_DIR/final_pkgs" diff --git a/cron/update_s3_htmls.sh b/cron/update_s3_htmls.sh deleted file mode 100755 index 2dbd172a5..000000000 --- a/cron/update_s3_htmls.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -set -e - -# Update the html links file in the s3 bucket Pip uses this html file to look -# through all the wheels and pick the most recently uploaded one (by the -# version, not the actual date of upload). There is one html file per cuda/cpu -# version - -# Upload for all CUDA/cpu versions if not given one to use -if [[ -z "$CUDA_VERSIONS" ]]; then - export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm5.0' 'rocm5.1.1') -fi - -if [[ -z "$HTML_NAME" ]]; then - export HTML_NAME='torch_nightly.html' -fi - -# Dry run disabled by default for legacy purposes -DRY_RUN=${DRY_RUN:-disabled} -DRY_RUN_FLAG="" -if [[ "${DRY_RUN}" != disabled ]]; then - DRY_RUN_FLAG="--dryrun" -fi - -# NB: includes trailing slash (from PIP_UPLOAD_FOLDER) -s3_base="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}" - -# Pull all existing whls in this directory and turn them into html links -# N.B. we use the .dev as a hacky way to exclude all wheels with old -# 'yyyy.mm.dd' versions -# -# NB: replacing + with %2B is to fix old versions of pip which don't -# this transform automatically. This makes the display a little -# ugly but whatever -function generate_html() { - # Trailing slash required in both cases - dir="$1" - url_prefix="$2" - aws s3 ls "${s3_base}${dir}" | grep --only-matching '\S*\.whl' | sed 's#+#%2B#g' | sed 's#.*#'"${url_prefix}"'&
#g' -} - -# This will be included in all the sub-indices -generate_html '' '../' > "root-$HTML_NAME" -generate_html '' '' > "$HTML_NAME" - -for cuda_ver in "${CUDA_VERSIONS[@]}"; do - generate_html "${cuda_ver}/" "" > "${cuda_ver}-$HTML_NAME" - cat "root-$HTML_NAME" >> "${cuda_ver}-$HTML_NAME" - generate_html "${cuda_ver}/" "${cuda_ver}/" >> "$HTML_NAME" - - # Check your work every once in a while - echo "Setting ${cuda_ver}/$HTML_NAME to:" - cat "${cuda_ver}-$HTML_NAME" - ( - set -x - aws s3 cp ${DRY_RUN_FLAG} "${cuda_ver}-$HTML_NAME" "s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate' - ) - -done - -# Check your work every once in a while -echo "Setting $HTML_NAME to:" -cat "$HTML_NAME" -( - set -x - - # Upload the html file back up - # Note the lack of a / b/c duplicate / do cause problems in s3 - aws s3 cp ${DRY_RUN_FLAG} "$HTML_NAME" "$s3_base$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate' -)