From c9fb8c7813624bce2af3df50689dc44349cd8864 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 13 May 2024 08:23:50 -0700 Subject: [PATCH] [BE] Remove unused files and dead code --- .circleci/scripts/binary_checkout.sh | 61 -------------------- .circleci/scripts/binary_populate_env.sh | 1 - cron/update_s3_htmls.sh | 71 ------------------------ manywheel/build_scripts/ssl-check.py | 33 ----------- manywheel/test_wheel.sh | 27 --------- 5 files changed, 193 deletions(-) delete mode 100755 .circleci/scripts/binary_checkout.sh delete mode 100755 cron/update_s3_htmls.sh delete mode 100644 manywheel/build_scripts/ssl-check.py delete mode 100755 manywheel/test_wheel.sh diff --git a/.circleci/scripts/binary_checkout.sh b/.circleci/scripts/binary_checkout.sh deleted file mode 100755 index b634f5c9a..000000000 --- a/.circleci/scripts/binary_checkout.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash -# Derived from https://github.com/pytorch/pytorch/blob/2c7df1360aa17d4a6d6726998eede3671bcb36ee/.circleci/scripts/binary_populate_env.sh - -set -eux -o pipefail - -retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) -} - - -# This step runs on multiple executors with different envfile locations -if [[ "$OSTYPE" == "msys" ]]; then - # windows executor (builds and tests) - rm -rf /c/w - ln -s "${HOME}" /c/w - WORK_DIR="/c/w" -elif [[ -d "/home/circleci/project" ]]; then - # machine executor (binary tests) - WORK_DIR="${HOME}/project" -else - # macos executor (builds and tests) - # docker executor (binary builds) - WORK_DIR="${HOME}" -fi - -if [[ "$OSTYPE" == "msys" ]]; then - # We need to make the paths as short as possible on Windows - PYTORCH_ROOT="$WORK_DIR/p" - BUILDER_ROOT="$WORK_DIR/b" -else - PYTORCH_ROOT="$WORK_DIR/pytorch" - BUILDER_ROOT="$WORK_DIR/builder" -fi - -# Persist these variables for the subsequent steps -echo "export WORK_DIR=${WORK_DIR}" >> ${BASH_ENV} -echo "export PYTORCH_ROOT=${PYTORCH_ROOT}" >> ${BASH_ENV} -echo "export BUILDER_ROOT=${BUILDER_ROOT}" >> ${BASH_ENV} - -# Clone the Pytorch branch -retry git clone --depth 1 https://github.com/pytorch/pytorch.git "$PYTORCH_ROOT" -# Removed checking out pytorch/pytorch using CIRCLE_PR_NUMBER and CIRCLE_SHA1 as -# those environment variables are tied to the host repo where the build is being -# triggered. -retry git submodule update --init --recursive -pushd "$PYTORCH_ROOT" -echo "Using Pytorch from " -git --no-pager log --max-count 1 -popd - -# Clone the Builder master repo -retry git clone -q https://github.com/pytorch/builder.git "$BUILDER_ROOT" -pushd "$BUILDER_ROOT" -if [[ -n "${CIRCLE_SHA1:-}" ]]; then - # Check out a specific commit (typically the latest) from pytorch/builder - git reset --hard "${CIRCLE_SHA1}" - git checkout -q -B main -fi -echo "Using builder from " -git --no-pager log --max-count 1 -popd diff --git a/.circleci/scripts/binary_populate_env.sh b/.circleci/scripts/binary_populate_env.sh index 7e663a64b..5b141ac38 100755 --- a/.circleci/scripts/binary_populate_env.sh +++ b/.circleci/scripts/binary_populate_env.sh @@ -143,7 +143,6 @@ export BUILD_JNI=$BUILD_JNI export PIP_UPLOAD_FOLDER="$PIP_UPLOAD_FOLDER" export DOCKER_IMAGE="$DOCKER_IMAGE" -# Remove WORKD_DIR, PYTORCH_ROOT, BUILDER_ROOT defined & persisted in binary_checkout.sh export MAC_PACKAGE_WORK_DIR="$WORK_DIR" export MINICONDA_ROOT="$WORK_DIR/miniconda" export PYTORCH_FINAL_PACKAGE_DIR="$WORK_DIR/final_pkgs" diff --git a/cron/update_s3_htmls.sh b/cron/update_s3_htmls.sh deleted file mode 100755 index 2dbd172a5..000000000 --- a/cron/update_s3_htmls.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -set -e - -# Update the html links file in the s3 bucket Pip uses this html file to look -# through all the wheels and pick the most recently uploaded one (by the -# version, not the actual date of upload). There is one html file per cuda/cpu -# version - -# Upload for all CUDA/cpu versions if not given one to use -if [[ -z "$CUDA_VERSIONS" ]]; then - export CUDA_VERSIONS=('cpu' 'cu92' 'cu100' 'cu101' 'cu102' 'cu110' 'rocm5.0' 'rocm5.1.1') -fi - -if [[ -z "$HTML_NAME" ]]; then - export HTML_NAME='torch_nightly.html' -fi - -# Dry run disabled by default for legacy purposes -DRY_RUN=${DRY_RUN:-disabled} -DRY_RUN_FLAG="" -if [[ "${DRY_RUN}" != disabled ]]; then - DRY_RUN_FLAG="--dryrun" -fi - -# NB: includes trailing slash (from PIP_UPLOAD_FOLDER) -s3_base="s3://pytorch/whl/${PIP_UPLOAD_FOLDER}" - -# Pull all existing whls in this directory and turn them into html links -# N.B. we use the .dev as a hacky way to exclude all wheels with old -# 'yyyy.mm.dd' versions -# -# NB: replacing + with %2B is to fix old versions of pip which don't -# this transform automatically. This makes the display a little -# ugly but whatever -function generate_html() { - # Trailing slash required in both cases - dir="$1" - url_prefix="$2" - aws s3 ls "${s3_base}${dir}" | grep --only-matching '\S*\.whl' | sed 's#+#%2B#g' | sed 's#.*#'"${url_prefix}"'&
#g' -} - -# This will be included in all the sub-indices -generate_html '' '../' > "root-$HTML_NAME" -generate_html '' '' > "$HTML_NAME" - -for cuda_ver in "${CUDA_VERSIONS[@]}"; do - generate_html "${cuda_ver}/" "" > "${cuda_ver}-$HTML_NAME" - cat "root-$HTML_NAME" >> "${cuda_ver}-$HTML_NAME" - generate_html "${cuda_ver}/" "${cuda_ver}/" >> "$HTML_NAME" - - # Check your work every once in a while - echo "Setting ${cuda_ver}/$HTML_NAME to:" - cat "${cuda_ver}-$HTML_NAME" - ( - set -x - aws s3 cp ${DRY_RUN_FLAG} "${cuda_ver}-$HTML_NAME" "s3://pytorch/whl/${PIP_UPLOAD_FOLDER}${cuda_ver}/$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate' - ) - -done - -# Check your work every once in a while -echo "Setting $HTML_NAME to:" -cat "$HTML_NAME" -( - set -x - - # Upload the html file back up - # Note the lack of a / b/c duplicate / do cause problems in s3 - aws s3 cp ${DRY_RUN_FLAG} "$HTML_NAME" "$s3_base$HTML_NAME" --acl public-read --cache-control 'no-cache,no-store,must-revalidate' -) diff --git a/manywheel/build_scripts/ssl-check.py b/manywheel/build_scripts/ssl-check.py deleted file mode 100644 index b91927173..000000000 --- a/manywheel/build_scripts/ssl-check.py +++ /dev/null @@ -1,33 +0,0 @@ -# cf. https://github.com/pypa/manylinux/issues/53 - -GOOD_SSL = "https://google.com" -BAD_SSL = "https://self-signed.badssl.com" - -import sys - -print("Testing SSL certificate checking for Python:", sys.version) - -if (sys.version_info[:2] < (2, 7) - or sys.version_info[:2] < (3, 4)): - print("This version never checks SSL certs; skipping tests") - sys.exit(0) - -if sys.version_info[0] >= 3: - from urllib.request import urlopen - EXC = OSError -else: - from urllib import urlopen - EXC = IOError - -print("Connecting to %s should work" % (GOOD_SSL,)) -urlopen(GOOD_SSL) -print("...it did, yay.") - -print("Connecting to %s should fail" % (BAD_SSL,)) -try: - urlopen(BAD_SSL) - # If we get here then we failed: - print("...it DIDN'T!!!!!11!!1one!") - sys.exit(1) -except EXC: - print("...it did, yay.") \ No newline at end of file diff --git a/manywheel/test_wheel.sh b/manywheel/test_wheel.sh deleted file mode 100755 index ada7d93f0..000000000 --- a/manywheel/test_wheel.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -yum install -y wget git - -rm -rf /usr/local/cuda* - -# Install Anaconda -if ! ls /py -then - echo "Miniconda needs to be installed" - wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh - bash ~/miniconda.sh -b -p /py -else - echo "Miniconda is already installed" -fi - -export PATH="/py/bin:$PATH" - -# Anaconda token -if ls /remote/token -then - source /remote/token -fi - -conda install -y conda-build anaconda-client -