Revert "[BE] Remove unused files and dead code"

.circleci/scripts/binary_checkout.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+# Derived from https://github.com/pytorch/pytorch/blob/2c7df1360aa17d4a6d6726998eede3671bcb36ee/.circleci/scripts/binary_populate_env.sh
+
+set -eux -o pipefail
+
+retry () {
+    $*  || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
+}
+
+
+# This step runs on multiple executors with different envfile locations
+if [[ "$OSTYPE" == "msys" ]]; then
+  # windows executor (builds and tests)
+  rm -rf /c/w
+  ln -s "${HOME}" /c/w
+  WORK_DIR="/c/w"
+elif [[ -d "/home/circleci/project" ]]; then
+  # machine executor (binary tests)
+  WORK_DIR="${HOME}/project"
+else
+  # macos executor (builds and tests)
+  # docker executor (binary builds)
+  WORK_DIR="${HOME}"
+fi
+
+if [[ "$OSTYPE" == "msys" ]]; then
+  # We need to make the paths as short as possible on Windows
+  PYTORCH_ROOT="$WORK_DIR/p"
+  BUILDER_ROOT="$WORK_DIR/b"
+else
+  PYTORCH_ROOT="$WORK_DIR/pytorch"
+  BUILDER_ROOT="$WORK_DIR/builder"
+fi
+
+# Persist these variables for the subsequent steps
+echo "export WORK_DIR=${WORK_DIR}" >> ${BASH_ENV}
+echo "export PYTORCH_ROOT=${PYTORCH_ROOT}" >> ${BASH_ENV}
+echo "export BUILDER_ROOT=${BUILDER_ROOT}" >> ${BASH_ENV}
+
+# Clone the Pytorch branch
+retry git clone --depth 1 https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT"
+# Removed checking out pytorch/pytorch using CIRCLE_PR_NUMBER and CIRCLE_SHA1 as
+# those environment variables are tied to the host repo where the build is being
+# triggered.
+retry git submodule update --init --recursive
+pushd "$PYTORCH_ROOT"
+echo "Using Pytorch from "
+git --no-pager log --max-count 1
+popd
+
+# Clone the Builder master repo
+retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT"
+pushd "$BUILDER_ROOT"
+if [[ -n "${CIRCLE_SHA1:-}" ]]; then
+  # Check out a specific commit (typically the latest) from pytorch/builder
+  git reset --hard "${CIRCLE_SHA1}"
+  git checkout -q -B main
+fi
+echo "Using builder from "
+git --no-pager log --max-count 1
+popd

.circleci/scripts/binary_populate_env.sh

@@ -143,6 +143,7 @@ export BUILD_JNI=$BUILD_JNI
 export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER"
 export DOCKER_IMAGE="$DOCKER_IMAGE"
 
+# Remove WORKD_DIR, PYTORCH_ROOT, BUILDER_ROOT defined & persisted in binary_checkout.sh
 export MAC_PACKAGE_WORK_DIR="$WORK_DIR"
 export MINICONDA_ROOT="$WORK_DIR/miniconda"
 export PYTORCH_FINAL_PACKAGE_DIR="$WORK_DIR/final_pkgs"

cron/update_s3_htmls.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+set -e
+
+# Update the html links file in the s3 bucket Pip uses this html file to look
+# through all the wheels and pick the most recently uploaded one (by the
+# version, not the actual date of upload). There is one html file per cuda/cpu
+# version
+
+# Upload for all CUDA/cpu versions if not given one to use
+if [[ -z "$CUDA_VERSIONS" ]]; then
+    export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm5.0' 'rocm5.1.1')
+fi
+
+if [[ -z "$HTML_NAME" ]]; then
+    export HTML_NAME='torch_nightly.html'
+fi
+
+# Dry run disabled by default for legacy purposes
+DRY_RUN=${DRY_RUN:-disabled}
+DRY_RUN_FLAG=""
+if [[ "${DRY_RUN}" != disabled ]]; then
+  DRY_RUN_FLAG="--dryrun"
+fi
+
+# NB: includes trailing slash (from PIP_UPLOAD_FOLDER)
+s3_base="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}"
+
+# Pull all existing whls in this directory and turn them into html links
+# N.B. we use the .dev as a hacky way to exclude all wheels with old
+# 'yyyy.mm.dd' versions
+#
+# NB: replacing + with %2B is to fix old versions of pip which don't
+# this transform automatically.  This makes the display a little
+# ugly but whatever
+function generate_html() {
+  # Trailing slash required in both cases
+  dir="$1"
+  url_prefix="$2"
+  aws s3 ls "${s3_base}${dir}" | grep --only-matching '\S*\.whl' | sed 's#+#%2B#g' | sed 's#.*#<a href="'"${url_prefix}"'&">'"${url_prefix}"'&</a><br>#g'
+}
+
+# This will be included in all the sub-indices
+generate_html '' '../' > "root-$HTML_NAME"
+generate_html '' '' > "$HTML_NAME"
+
+for cuda_ver in "${CUDA_VERSIONS[@]}"; do
+    generate_html "${cuda_ver}/" "" > "${cuda_ver}-$HTML_NAME"
+    cat "root-$HTML_NAME" >> "${cuda_ver}-$HTML_NAME"
+    generate_html "${cuda_ver}/" "${cuda_ver}/" >> "$HTML_NAME"
+
+    # Check your work every once in a while
+    echo "Setting ${cuda_ver}/$HTML_NAME to:"
+    cat "${cuda_ver}-$HTML_NAME"
+    (
+      set -x
+      aws s3 cp ${DRY_RUN_FLAG} "${cuda_ver}-$HTML_NAME" "s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/$HTML_NAME"  --acl public-read --cache-control 'no-cache,no-store,must-revalidate'
+    )
+
+done
+
+# Check your work every once in a while
+echo "Setting $HTML_NAME to:"
+cat "$HTML_NAME"
+(
+  set -x
+
+  # Upload the html file back up
+  # Note the lack of a / b/c duplicate / do cause problems in s3
+  aws s3 cp ${DRY_RUN_FLAG} "$HTML_NAME" "$s3_base$HTML_NAME"  --acl public-read --cache-control 'no-cache,no-store,must-revalidate'
+)

manywheel/build_scripts/ssl-check.py

@@ -0,0 +1,33 @@
+# cf. https://github.com/pypa/manylinux/issues/53
+
+GOOD_SSL = "https://google.com"
+BAD_SSL = "https://self-signed.badssl.com"
+
+import sys
+
+print("Testing SSL certificate checking for Python:", sys.version)
+
+if (sys.version_info[:2] < (2, 7)
+    or sys.version_info[:2] < (3, 4)):
+    print("This version never checks SSL certs; skipping tests")
+    sys.exit(0)
+
+if sys.version_info[0] >= 3:
+    from urllib.request import urlopen
+    EXC = OSError
+else:
+    from urllib import urlopen
+    EXC = IOError
+
+print("Connecting to %s should work" % (GOOD_SSL,))
+urlopen(GOOD_SSL)
+print("...it did, yay.")
+
+print("Connecting to %s should fail" % (BAD_SSL,))
+try:
+    urlopen(BAD_SSL)
+    # If we get here then we failed:
+    print("...it DIDN'T!!!!!11!!1one!")
+    sys.exit(1)
+except EXC:
+    print("...it did, yay.")

manywheel/test_wheel.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -e
+
+yum install -y wget git
+
+rm -rf /usr/local/cuda*
+
+# Install Anaconda
+if ! ls /py
+then
+    echo "Miniconda needs to be installed"
+    wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
+    bash ~/miniconda.sh -b -p /py
+else
+    echo "Miniconda is already installed"
+fi
+
+export PATH="/py/bin:$PATH"
+
+# Anaconda token
+if ls /remote/token
+then
+   source /remote/token
+fi
+
+conda install -y conda-build anaconda-client
+