From f3b676a1921346a55d1564ec8ca1270a69cb06fd Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 17 Oct 2022 11:09:07 -0700 Subject: [PATCH 1/4] Validate python 3.11 --- .github/actions/validate-binary/action.yml | 7 +- .github/workflows/validate-linux-binaries.yml | 4 +- test/smoke_test/smoke_test.py | 105 ++++++++++-------- 3 files changed, 66 insertions(+), 50 deletions(-) diff --git a/.github/actions/validate-binary/action.yml b/.github/actions/validate-binary/action.yml index 41bcca545..8093dfcd8 100644 --- a/.github/actions/validate-binary/action.yml +++ b/.github/actions/validate-binary/action.yml @@ -69,7 +69,12 @@ runs: set -ex conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy conda run -p ${ENV_NAME} $INSTALLATION - conda run -p ${ENV_NAME} python3 ./test/smoke_test/smoke_test.py + if [ $DESIRED_PYTHON == "3.11" ]; then + conda run -p ${ENV_NAME} python3 --package torchonly ./test/smoke_test/smoke_test.py + else + conda run -p ${ENV_NAME} python3 ./test/smoke_test/smoke_test.py + fi + export LD_LIBRARY_PATH="$(dirname $(which python))/lib" export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml index 70daec4a4..e1a476858 100644 --- a/.github/workflows/validate-linux-binaries.yml +++ b/.github/workflows/validate-linux-binaries.yml @@ -67,8 +67,10 @@ jobs: fail-fast: false runs-on: ${{ matrix.validation_runner }} steps: + - name: Checkout PyTorch builder + uses: actions/checkout@v2 - name: Validate binary wheel - uses: pytorch/builder/.github/actions/validate-binary@main + uses: .github/actions/validate-binary with: gpu_arch_type: ${{ matrix.gpu_arch_type }} gpu_arch_ver: ${{ matrix.gpu_arch_version }} diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index e069a712e..e4646a76e 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -2,14 +2,10 @@ import re import sys from pathlib import Path - +import argparse import torch -import torchaudio -# the following import would invoke -# _check_cuda_version() -# via torchvision.extension._check_cuda_version() -import torchvision + gpu_arch_ver = os.getenv("GPU_ARCH_VER") gpu_arch_type = os.getenv("GPU_ARCH_TYPE") @@ -38,20 +34,21 @@ def get_anaconda_output_for_package(pkg_name_str): return output.strip().split('\n')[-1] -def check_nightly_binaries_date() -> None: +def check_nightly_binaries_date(package: str) -> None: torch_str = torch.__version__ - ta_str = torchaudio.__version__ - tv_str = torchvision.__version__ - date_t_str = re.findall("dev\d+", torch.__version__) - date_ta_str = re.findall("dev\d+", torchaudio.__version__) - date_tv_str = re.findall("dev\d+", torchvision.__version__) - # check that the above three lists are equal and none of them is empty - if not date_t_str or not date_t_str == date_ta_str == date_tv_str: - raise RuntimeError( - f"Expected torch, torchaudio, torchvision to be the same date. But they are from {date_t_str}, {date_ta_str}, {date_tv_str} respectively" - ) + if(package == "all"): + ta_str = torchaudio.__version__ + tv_str = torchvision.__version__ + date_ta_str = re.findall("dev\d+", torchaudio.__version__) + date_tv_str = re.findall("dev\d+", torchvision.__version__) + + # check that the above three lists are equal and none of them is empty + if not date_t_str or not date_t_str == date_ta_str == date_tv_str: + raise RuntimeError( + f"Expected torch, torchaudio, torchvision to be the same date. But they are from {date_t_str}, {date_ta_str}, {date_tv_str} respectively" + ) # check that the date is recent, at this point, date_torch_str is not empty binary_date_str = date_t_str[0][3:] @@ -65,8 +62,7 @@ def check_nightly_binaries_date() -> None: f"the binaries are from {binary_date_obj} and are more than 2 days old!" ) - -def smoke_test_cuda() -> None: +def smoke_test_cuda(package: str) -> None: if not torch.cuda.is_available() and is_cuda_system: raise RuntimeError(f"Expected CUDA {gpu_arch_ver}. However CUDA is not loaded.") if torch.cuda.is_available(): @@ -79,23 +75,23 @@ def smoke_test_cuda() -> None: print(f"torch cudnn: {torch.backends.cudnn.version()}") print(f"cuDNN enabled? {torch.backends.cudnn.enabled}") - if installation_str.find("nightly") != -1: - # just print out cuda version, as version check were already performed during import - print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}") - print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}") - else: - # torchaudio runtime added the cuda verison check on 09/23/2022 via - # https://github.com/pytorch/audio/pull/2707 - # so relying on anaconda output for pytorch-test and pytorch channel - torchaudio_allstr = get_anaconda_output_for_package(torchaudio.__name__) - if ( - is_cuda_system - and "cu" + str(gpu_arch_ver).replace(".", "") not in torchaudio_allstr - ): - raise RuntimeError( - f"CUDA version issue. Loaded: {torchaudio_allstr} Expected: {gpu_arch_ver}" - ) - + if(package == 'all'): + if installation_str.find("nightly") != -1: + # just print out cuda version, as version check were already performed during import + print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}") + print(f"torchaudio cuda: {torch.ops.torchaudio.cuda_version()}") + else: + # torchaudio runtime added the cuda verison check on 09/23/2022 via + # https://github.com/pytorch/audio/pull/2707 + # so relying on anaconda output for pytorch-test and pytorch channel + torchaudio_allstr = get_anaconda_output_for_package(torchaudio.__name__) + if ( + is_cuda_system + and "cu" + str(gpu_arch_ver).replace(".", "") not in torchaudio_allstr + ): + raise RuntimeError( + f"CUDA version issue. Loaded: {torchaudio_allstr} Expected: {gpu_arch_ver}" + ) def smoke_test_conv2d() -> None: import torch.nn as nn @@ -180,24 +176,37 @@ def smoke_test_torchaudio() -> None: def main() -> None: - # todo add torch, torchvision and torchaudio tests + parser = argparse.ArgumentParser() + parser.add_argument( + "--package", + help="Package to include in smoke testing", + type=str, + choices=["all", "torchonly"], + default="all", + ) + print(f"torch: {torch.__version__}") - print(f"torchvision: {torchvision.__version__}") - print(f"torchaudio: {torchaudio.__version__}") - smoke_test_cuda() + smoke_test_cuda(options.package) + smoke_test_conv2d() # only makes sense to check nightly package where dates are known if installation_str.find("nightly") != -1: check_nightly_binaries_date() - smoke_test_conv2d() - smoke_test_torchaudio() - smoke_test_torchvision() - smoke_test_torchvision_read_decode() - smoke_test_torchvision_resnet50_classify() - if torch.cuda.is_available(): - smoke_test_torchvision_resnet50_classify("cuda") - + if options.package == "all": + import torchaudio + # the following import would invoke + # _check_cuda_version() + # via torchvision.extension._check_cuda_version() + import torchvision + print(f"torchvision: {torchvision.__version__}") + print(f"torchaudio: {torchaudio.__version__}") + smoke_test_torchaudio() + smoke_test_torchvision() + smoke_test_torchvision_read_decode() + smoke_test_torchvision_resnet50_classify() + if torch.cuda.is_available(): + smoke_test_torchvision_resnet50_classify("cuda") if __name__ == "__main__": main() From 2067561d8ffa09b0e1c6b37fbd4303dc1d29d614 Mon Sep 17 00:00:00 2001 From: atalman Date: Mon, 17 Oct 2022 11:21:38 -0700 Subject: [PATCH 2/4] Validate linux binaries change Add options Import torchvision Adding python 3.11 install pass package to check nightly binaries date Test test Add python 3.11 code testing Adding python 3.11 test Add python 3.11 validation Adding zlib develop install Install zlib etc.. Adding zlib1g as well testing testing Adding validate windows binary Trying to workaround testing Refacor smoke test Add import statement fix datetime call --- .github/actions/validate-binary/action.yml | 35 +++++++++++------ .github/workflows/validate-linux-binaries.yml | 3 +- test/smoke_test/smoke_test.py | 39 +++++++++---------- 3 files changed, 45 insertions(+), 32 deletions(-) diff --git a/.github/actions/validate-binary/action.yml b/.github/actions/validate-binary/action.yml index 8093dfcd8..ab3ec4728 100644 --- a/.github/actions/validate-binary/action.yml +++ b/.github/actions/validate-binary/action.yml @@ -44,7 +44,7 @@ runs: run: | nvidia-smi - name: Install Conda Linux - if: ${{ inputs.target_os == 'linux' }} + if: ${{ inputs.target_os == 'linux' && inputs.python_version != '3.11' }} uses: conda-incubator/setup-miniconda@v2 with: python-version: ${{ inputs.python_version }} @@ -66,16 +66,29 @@ runs: DESIRED_DEVTOOLSET: ${{ inputs.dev_toolset }} PACKAGE_TYPE: ${{ inputs.package_type }} run: | - set -ex - conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy - conda run -p ${ENV_NAME} $INSTALLATION - if [ $DESIRED_PYTHON == "3.11" ]; then - conda run -p ${ENV_NAME} python3 --package torchonly ./test/smoke_test/smoke_test.py + + if [ $DESIRED_PYTHON == '3.11' ]; then + set -ex + export CPYTHON_VERSIONS=3.11.0 + sudo apt-get install build-essential gdb lcov libbz2-dev libffi-dev \ + libgdbm-dev liblzma-dev libncurses5-dev libreadline6-dev \ + libsqlite3-dev libssl-dev lzma lzma-dev tk-dev uuid-dev zlib1g zlib1g-dev -y + + export PYTHON_PATH="/opt/_internal/cpython-3.11.0rc2/bin/" + export PIP_PATH="${PYTHON_PATH}/pip" + export PIP_INSTALLATION="${INSTALLATION/pip3/"$PIP_PATH"}" + ./common/install_cpython.sh + + eval ${PYTHON_PATH}/python --version + eval ${PIP_INSTALLATION} + eval ${PYTHON_PATH}/python ./test/smoke_test/smoke_test.py --package torchonly else + set -ex + conda create -yp ${ENV_NAME} python=${{ inputs.python_version }} numpy + conda run -p ${ENV_NAME} $INSTALLATION conda run -p ${ENV_NAME} python3 ./test/smoke_test/smoke_test.py + export LD_LIBRARY_PATH="$(dirname $(which python))/lib" + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib + conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh + conda env remove -p ${ENV_NAME} fi - - export LD_LIBRARY_PATH="$(dirname $(which python))/lib" - export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/cuda/lib - conda run -p ${ENV_NAME} env LD_LIBRARY_PATH=$LD_LIBRARY_PATH bash ${PWD}/check_binary.sh - conda env remove -p ${ENV_NAME} diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml index e1a476858..eb5cd9b3f 100644 --- a/.github/workflows/validate-linux-binaries.yml +++ b/.github/workflows/validate-linux-binaries.yml @@ -32,6 +32,7 @@ jobs: package-type: wheel os: linux channel: ${{ inputs.channel }} + with-py311: enable generate-linux-libtorch-matrix: uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main with: @@ -70,7 +71,7 @@ jobs: - name: Checkout PyTorch builder uses: actions/checkout@v2 - name: Validate binary wheel - uses: .github/actions/validate-binary + uses: ./.github/actions/validate-binary with: gpu_arch_type: ${{ matrix.gpu_arch_type }} gpu_arch_ver: ${{ matrix.gpu_arch_version }} diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index e4646a76e..1adcd4459 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -5,14 +5,13 @@ import argparse import torch - - gpu_arch_ver = os.getenv("GPU_ARCH_VER") gpu_arch_type = os.getenv("GPU_ARCH_TYPE") # use installation env variable to tell if it is nightly channel installation_str = os.getenv("INSTALLATION") is_cuda_system = gpu_arch_type == "cuda" SCRIPT_DIR = Path(__file__).parent +NIGHTLY_ALLOWED_DELTA = 3 # helper function to return the conda installed packages # and return package we are insterseted in @@ -35,32 +34,31 @@ def get_anaconda_output_for_package(pkg_name_str): def check_nightly_binaries_date(package: str) -> None: + from datetime import datetime, timedelta + format_dt = '%Y%m%d' + torch_str = torch.__version__ date_t_str = re.findall("dev\d+", torch.__version__) + date_t_delta = datetime.now() - datetime.strptime(date_t_str.lstrip("dev"), format_dt) + if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA: + raise RuntimeError( + f"the binaries are from {date_t_str} and are more than {NIGHTLY_ALLOWED_DELTA} days old!" + ) if(package == "all"): ta_str = torchaudio.__version__ tv_str = torchvision.__version__ date_ta_str = re.findall("dev\d+", torchaudio.__version__) date_tv_str = re.findall("dev\d+", torchvision.__version__) + date_ta_delta = datetime.now() - datetime.strptime(date_ta_str.lstrip("dev"), format_dt) + date_tv_delta = datetime.now() - datetime.strptime(date_tv_str.lstrip("dev"), format_dt) # check that the above three lists are equal and none of them is empty - if not date_t_str or not date_t_str == date_ta_str == date_tv_str: + if date_ta_delta.days > NIGHTLY_ALLOWED_DELTA or date_tv_delta.days > NIGHTLY_ALLOWED_DELTA: raise RuntimeError( - f"Expected torch, torchaudio, torchvision to be the same date. But they are from {date_t_str}, {date_ta_str}, {date_tv_str} respectively" + f"Expected torchaudio, torchvision to be less then {NIGHTLY_ALLOWED_DELTA} days. But they are from {date_ta_str}, {date_tv_str} respectively" ) - # check that the date is recent, at this point, date_torch_str is not empty - binary_date_str = date_t_str[0][3:] - from datetime import datetime - - binary_date_obj = datetime.strptime(binary_date_str, "%Y%m%d").date() - today_obj = datetime.today().date() - delta = today_obj - binary_date_obj - if delta.days >= 2: - raise RuntimeError( - f"the binaries are from {binary_date_obj} and are more than 2 days old!" - ) def smoke_test_cuda(package: str) -> None: if not torch.cuda.is_available() and is_cuda_system: @@ -76,6 +74,8 @@ def smoke_test_cuda(package: str) -> None: print(f"cuDNN enabled? {torch.backends.cudnn.enabled}") if(package == 'all'): + import torchaudio + import torchvision if installation_str.find("nightly") != -1: # just print out cuda version, as version check were already performed during import print(f"torchvision cuda: {torch.ops.torchvision._cuda_version()}") @@ -165,6 +165,7 @@ def smoke_test_torchvision_resnet50_classify(device: str = "cpu") -> None: def smoke_test_torchaudio() -> None: + import torchaudio import torchaudio.compliance.kaldi # noqa: F401 import torchaudio.datasets # noqa: F401 import torchaudio.functional # noqa: F401 @@ -184,20 +185,18 @@ def main() -> None: choices=["all", "torchonly"], default="all", ) - + options = parser.parse_args() print(f"torch: {torch.__version__}") + smoke_test_cuda(options.package) smoke_test_conv2d() # only makes sense to check nightly package where dates are known if installation_str.find("nightly") != -1: - check_nightly_binaries_date() + check_nightly_binaries_date(options.package) if options.package == "all": import torchaudio - # the following import would invoke - # _check_cuda_version() - # via torchvision.extension._check_cuda_version() import torchvision print(f"torchvision: {torchvision.__version__}") print(f"torchaudio: {torchaudio.__version__}") From c7452fd32c25bb5ad78c120347777396298af6cf Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 19 Oct 2022 07:57:51 -0700 Subject: [PATCH 3/4] Fix stripping dev --- test/smoke_test/smoke_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 1adcd4459..f1f52a547 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -39,7 +39,7 @@ def check_nightly_binaries_date(package: str) -> None: torch_str = torch.__version__ date_t_str = re.findall("dev\d+", torch.__version__) - date_t_delta = datetime.now() - datetime.strptime(date_t_str.lstrip("dev"), format_dt) + date_t_delta = datetime.now() - datetime.strptime(date_t_str[0][3:], format_dt) if date_t_delta.days >= NIGHTLY_ALLOWED_DELTA: raise RuntimeError( f"the binaries are from {date_t_str} and are more than {NIGHTLY_ALLOWED_DELTA} days old!" @@ -50,8 +50,8 @@ def check_nightly_binaries_date(package: str) -> None: tv_str = torchvision.__version__ date_ta_str = re.findall("dev\d+", torchaudio.__version__) date_tv_str = re.findall("dev\d+", torchvision.__version__) - date_ta_delta = datetime.now() - datetime.strptime(date_ta_str.lstrip("dev"), format_dt) - date_tv_delta = datetime.now() - datetime.strptime(date_tv_str.lstrip("dev"), format_dt) + date_ta_delta = datetime.now() - datetime.strptime(date_ta_str[0][3:], format_dt) + date_tv_delta = datetime.now() - datetime.strptime(date_tv_str[0][3:], format_dt) # check that the above three lists are equal and none of them is empty if date_ta_delta.days > NIGHTLY_ALLOWED_DELTA or date_tv_delta.days > NIGHTLY_ALLOWED_DELTA: From bc7467e75bdca8d626006ea2c8a470b0ff1a7721 Mon Sep 17 00:00:00 2001 From: atalman Date: Wed, 19 Oct 2022 08:15:51 -0700 Subject: [PATCH 4/4] fix import --- test/smoke_test/smoke_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index f1f52a547..0ddcd267a 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -46,6 +46,8 @@ def check_nightly_binaries_date(package: str) -> None: ) if(package == "all"): + import torchaudio + import torchvision ta_str = torchaudio.__version__ tv_str = torchvision.__version__ date_ta_str = re.findall("dev\d+", torchaudio.__version__)