From 5e51c80e3dabc7d4dfc2205a9d66bddfae97020c Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Mon, 13 May 2024 16:09:36 -0400 Subject: [PATCH] Revert "[BE] Remove unused files and dead code (#1819)" This reverts commit bebc062488523afe62b8ec90ee91455316448406. --- .circleci/scripts/binary_checkout.sh | 61 ++++++++++++++++++++ .circleci/scripts/binary_populate_env.sh | 1 + cron/update_s3_htmls.sh | 71 ++++++++++++++++++++++++ manywheel/build_scripts/ssl-check.py | 33 +++++++++++ manywheel/test_wheel.sh | 27 +++++++++ 5 files changed, 193 insertions(+) create mode 100755 .circleci/scripts/binary_checkout.sh create mode 100755 cron/update_s3_htmls.sh create mode 100644 manywheel/build_scripts/ssl-check.py create mode 100755 manywheel/test_wheel.sh diff --git a/.circleci/scripts/binary_checkout.sh b/.circleci/scripts/binary_checkout.sh new file mode 100755 index 000000000..b634f5c9a --- /dev/null +++ b/.circleci/scripts/binary_checkout.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Derived from https://github.com/pytorch/pytorch/blob/2c7df1360aa17d4a6d6726998eede3671bcb36ee/.circleci/scripts/binary_populate_env.sh + +set -eux -o pipefail + +retry () { + $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) +} + + +# This step runs on multiple executors with different envfile locations +if [[ "$OSTYPE" == "msys" ]]; then + # windows executor (builds and tests) + rm -rf /c/w + ln -s "${HOME}" /c/w + WORK_DIR="/c/w" +elif [[ -d "/home/circleci/project" ]]; then + # machine executor (binary tests) + WORK_DIR="${HOME}/project" +else + # macos executor (builds and tests) + # docker executor (binary builds) + WORK_DIR="${HOME}" +fi + +if [[ "$OSTYPE" == "msys" ]]; then + # We need to make the paths as short as possible on Windows + PYTORCH_ROOT="$WORK_DIR/p" + BUILDER_ROOT="$WORK_DIR/b" +else + PYTORCH_ROOT="$WORK_DIR/pytorch" + BUILDER_ROOT="$WORK_DIR/builder" +fi + +# Persist these variables for the subsequent steps +echo "export WORK_DIR=${WORK_DIR}" >> ${BASH_ENV} +echo "export PYTORCH_ROOT=${PYTORCH_ROOT}" >> ${BASH_ENV} +echo "export BUILDER_ROOT=${BUILDER_ROOT}" >> ${BASH_ENV} + +# Clone the Pytorch branch +retry git clone --depth 1 https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT" +# Removed checking out pytorch/pytorch using CIRCLE_PR_NUMBER and CIRCLE_SHA1 as +# those environment variables are tied to the host repo where the build is being +# triggered. +retry git submodule update --init --recursive +pushd "$PYTORCH_ROOT" +echo "Using Pytorch from " +git --no-pager log --max-count 1 +popd + +# Clone the Builder master repo +retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT" +pushd "$BUILDER_ROOT" +if [[ -n "${CIRCLE_SHA1:-}" ]]; then + # Check out a specific commit (typically the latest) from pytorch/builder + git reset --hard "${CIRCLE_SHA1}" + git checkout -q -B main +fi +echo "Using builder from " +git --no-pager log --max-count 1 +popd diff --git a/.circleci/scripts/binary_populate_env.sh b/.circleci/scripts/binary_populate_env.sh index 5b141ac38..7e663a64b 100755 --- a/.circleci/scripts/binary_populate_env.sh +++ b/.circleci/scripts/binary_populate_env.sh @@ -143,6 +143,7 @@ export BUILD_JNI=$BUILD_JNI export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER" export DOCKER_IMAGE="$DOCKER_IMAGE" +# Remove WORKD_DIR, PYTORCH_ROOT, BUILDER_ROOT defined & persisted in binary_checkout.sh export MAC_PACKAGE_WORK_DIR="$WORK_DIR" export MINICONDA_ROOT="$WORK_DIR/miniconda" export PYTORCH_FINAL_PACKAGE_DIR="$WORK_DIR/final_pkgs" diff --git a/cron/update_s3_htmls.sh b/cron/update_s3_htmls.sh new file mode 100755 index 000000000..2dbd172a5 --- /dev/null +++ b/cron/update_s3_htmls.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +# Update the html links file in the s3 bucket Pip uses this html file to look +# through all the wheels and pick the most recently uploaded one (by the +# version, not the actual date of upload). There is one html file per cuda/cpu +# version + +# Upload for all CUDA/cpu versions if not given one to use +if [[ -z "$CUDA_VERSIONS" ]]; then + export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm5.0' 'rocm5.1.1') +fi + +if [[ -z "$HTML_NAME" ]]; then + export HTML_NAME='torch_nightly.html' +fi + +# Dry run disabled by default for legacy purposes +DRY_RUN=${DRY_RUN:-disabled} +DRY_RUN_FLAG="" +if [[ "${DRY_RUN}" != disabled ]]; then + DRY_RUN_FLAG="--dryrun" +fi + +# NB: includes trailing slash (from PIP_UPLOAD_FOLDER) +s3_base="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}" + +# Pull all existing whls in this directory and turn them into html links +# N.B. we use the .dev as a hacky way to exclude all wheels with old +# 'yyyy.mm.dd' versions +# +# NB: replacing + with %2B is to fix old versions of pip which don't +# this transform automatically. This makes the display a little +# ugly but whatever +function generate_html() { + # Trailing slash required in both cases + dir="$1" + url_prefix="$2" + aws s3 ls "${s3_base}${dir}" | grep --only-matching '\S*\.whl' | sed 's#+#%2B#g' | sed 's#.*#'"${url_prefix}"'&
#g' +} + +# This will be included in all the sub-indices +generate_html '' '../' > "root-$HTML_NAME" +generate_html '' '' > "$HTML_NAME" + +for cuda_ver in "${CUDA_VERSIONS[@]}"; do + generate_html "${cuda_ver}/" "" > "${cuda_ver}-$HTML_NAME" + cat "root-$HTML_NAME" >> "${cuda_ver}-$HTML_NAME" + generate_html "${cuda_ver}/" "${cuda_ver}/" >> "$HTML_NAME" + + # Check your work every once in a while + echo "Setting ${cuda_ver}/$HTML_NAME to:" + cat "${cuda_ver}-$HTML_NAME" + ( + set -x + aws s3 cp ${DRY_RUN_FLAG} "${cuda_ver}-$HTML_NAME" "s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate' + ) + +done + +# Check your work every once in a while +echo "Setting $HTML_NAME to:" +cat "$HTML_NAME" +( + set -x + + # Upload the html file back up + # Note the lack of a / b/c duplicate / do cause problems in s3 + aws s3 cp ${DRY_RUN_FLAG} "$HTML_NAME" "$s3_base$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate' +) diff --git a/manywheel/build_scripts/ssl-check.py b/manywheel/build_scripts/ssl-check.py new file mode 100644 index 000000000..b91927173 --- /dev/null +++ b/manywheel/build_scripts/ssl-check.py @@ -0,0 +1,33 @@ +# cf. https://github.com/pypa/manylinux/issues/53 + +GOOD_SSL = "https://google.com" +BAD_SSL = "https://self-signed.badssl.com" + +import sys + +print("Testing SSL certificate checking for Python:", sys.version) + +if (sys.version_info[:2] < (2, 7) + or sys.version_info[:2] < (3, 4)): + print("This version never checks SSL certs; skipping tests") + sys.exit(0) + +if sys.version_info[0] >= 3: + from urllib.request import urlopen + EXC = OSError +else: + from urllib import urlopen + EXC = IOError + +print("Connecting to %s should work" % (GOOD_SSL,)) +urlopen(GOOD_SSL) +print("...it did, yay.") + +print("Connecting to %s should fail" % (BAD_SSL,)) +try: + urlopen(BAD_SSL) + # If we get here then we failed: + print("...it DIDN'T!!!!!11!!1one!") + sys.exit(1) +except EXC: + print("...it did, yay.") \ No newline at end of file diff --git a/manywheel/test_wheel.sh b/manywheel/test_wheel.sh new file mode 100755 index 000000000..ada7d93f0 --- /dev/null +++ b/manywheel/test_wheel.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -e + +yum install -y wget git + +rm -rf /usr/local/cuda* + +# Install Anaconda +if ! ls /py +then + echo "Miniconda needs to be installed" + wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh + bash ~/miniconda.sh -b -p /py +else + echo "Miniconda is already installed" +fi + +export PATH="/py/bin:$PATH" + +# Anaconda token +if ls /remote/token +then + source /remote/token +fi + +conda install -y conda-build anaconda-client +