From 07dcc235329440e1d30d32be48c4e4d36e02d824 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:20:11 -0700 Subject: [PATCH 1/8] Fix wheel validations --- .github/scripts/validate_binaries.sh | 5 +++++ .github/workflows/validate-nightly-binaries.yml | 10 +--------- .github/workflows/validate-release-binaries.yml | 9 +++++++++ 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 88a76de86..6502ba629 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -6,6 +6,11 @@ else conda create -y -n ${ENV_NAME} python=${MATRIX_PYTHON_VERSION} numpy ffmpeg conda activate ${ENV_NAME} INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"} + + # Make sure no cache dir is specified during wheel + if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then + INSTALLATION=${INSTALLATION/"install"/"install --no-cache-dir"} + fi eval $INSTALLATION if [[ ${TARGET_OS} == 'linux' ]]; then diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml index 4f59d3e9f..f3f54180b 100644 --- a/.github/workflows/validate-nightly-binaries.yml +++ b/.github/workflows/validate-nightly-binaries.yml @@ -17,15 +17,7 @@ on: - .github/workflows/validate-macos-binaries.yml - .github/workflows/validate-macos-arm64-binaries.yml - test/smoke_test/* - pull_request: - paths: - - .github/workflows/validate-nightly-binaries.yml - - .github/workflows/validate-linux-binaries.yml - - .github/workflows/validate-windows-binaries.yml - - .github/workflows/validate-macos-binaries.yml - - .github/workflows/validate-macos-arm64-binaries.yml - - .github/scripts/validate_binaries.sh - - test/smoke_test/* + jobs: nightly: uses: ./.github/workflows/validate-binaries.yml diff --git a/.github/workflows/validate-release-binaries.yml b/.github/workflows/validate-release-binaries.yml index 73a8071ee..eb77e7d53 100644 --- a/.github/workflows/validate-release-binaries.yml +++ b/.github/workflows/validate-release-binaries.yml @@ -17,6 +17,15 @@ on: - .github/workflows/validate-macos-binaries.yml - .github/workflows/validate-macos-arm64-binaries.yml - test/smoke_test/* + pull_request: + paths: + - .github/workflows/validate-nightly-binaries.yml + - .github/workflows/validate-linux-binaries.yml + - .github/workflows/validate-windows-binaries.yml + - .github/workflows/validate-macos-binaries.yml + - .github/workflows/validate-macos-arm64-binaries.yml + - .github/scripts/validate_binaries.sh + - test/smoke_test/* jobs: release: From 38fc3bc5c70869b52fac1c127d145cc728fe9803 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:24:12 -0700 Subject: [PATCH 2/8] Try using upgrade flag instead --- .github/scripts/validate_binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 6502ba629..ebc68d279 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -9,7 +9,7 @@ else # Make sure no cache dir is specified during wheel if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then - INSTALLATION=${INSTALLATION/"install"/"install --no-cache-dir"} + INSTALLATION=${INSTALLATION/"install"/"install --upgrade"} fi eval $INSTALLATION From 1b9c6a94b3513979492e87a3482147f1c563ddde Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:29:02 -0700 Subject: [PATCH 3/8] try uninstall --- .github/scripts/validate_binaries.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index ebc68d279..ee907787f 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -7,9 +7,10 @@ else conda activate ${ENV_NAME} INSTALLATION=${MATRIX_INSTALLATION/"conda install"/"conda install -y"} - # Make sure no cache dir is specified during wheel + # Make sure we remove previous installation if it exists if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then - INSTALLATION=${INSTALLATION/"install"/"install --upgrade"} + UNINSTALL=${INSTALLATION/"install"/"uninstall"} + eval $UNINSTALL fi eval $INSTALLATION From 8d99967a33dceba7f3a7bc3aff66bfe164360639 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:32:32 -0700 Subject: [PATCH 4/8] test --- .github/scripts/validate_binaries.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index ee907787f..5a9813a6c 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -9,7 +9,7 @@ else # Make sure we remove previous installation if it exists if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then - UNINSTALL=${INSTALLATION/"install"/"uninstall"} + UNINSTALL=${INSTALLATION/"install"/"uninstall -y"} eval $UNINSTALL fi eval $INSTALLATION From 478252918d16d7e50a0eb995f6ab955d5e2ccca1 Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:38:40 -0700 Subject: [PATCH 5/8] Try using python3 --- .github/scripts/validate_binaries.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 5a9813a6c..87a4d96e1 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -20,7 +20,12 @@ else ${PWD}/check_binary.sh fi - python ./test/smoke_test/smoke_test.py + if [[ ${TARGET_OS} == 'windows' ]]; then + python ./test/smoke_test/smoke_test.py + else + python3 ./test/smoke_test/smoke_test.py + fi + conda deactivate conda env remove -n ${ENV_NAME} fi From 30f38781768c1a56b7393e9877fc448e32426e2f Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:43:47 -0700 Subject: [PATCH 6/8] use python3 vs python for validation --- .github/scripts/validate_binaries.sh | 7 +------ test/smoke_test/smoke_test.py | 4 ++-- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 87a4d96e1..66fb4f78d 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -20,12 +20,7 @@ else ${PWD}/check_binary.sh fi - if [[ ${TARGET_OS} == 'windows' ]]; then - python ./test/smoke_test/smoke_test.py - else - python3 ./test/smoke_test/smoke_test.py - fi - + python3 ./test/smoke_test/smoke_test.py conda deactivate conda env remove -n ${ENV_NAME} fi diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 34aaabb8e..226559914 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -24,14 +24,14 @@ { "name": "torchvision", "repo": "https://github.com/pytorch/vision.git", - "smoke_test": "python ./vision/test/smoke_test.py", + "smoke_test": "python3 ./vision/test/smoke_test.py", "extension": "extension", "repo_name": "vision", }, { "name": "torchaudio", "repo": "https://github.com/pytorch/audio.git", - "smoke_test": "python ./audio/test/smoke_test/smoke_test.py --no-ffmpeg", + "smoke_test": "python3 ./audio/test/smoke_test/smoke_test.py --no-ffmpeg", "extension": "_extension", "repo_name": "audio", }, From a7624e5c045d63bf62134767f664beb6ed53164e Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 13:57:49 -0700 Subject: [PATCH 7/8] Fix windows vs other os python execution --- .github/scripts/validate_binaries.sh | 7 ++++++- test/smoke_test/smoke_test.py | 10 +++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 66fb4f78d..87a4d96e1 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -20,7 +20,12 @@ else ${PWD}/check_binary.sh fi - python3 ./test/smoke_test/smoke_test.py + if [[ ${TARGET_OS} == 'windows' ]]; then + python ./test/smoke_test/smoke_test.py + else + python3 ./test/smoke_test/smoke_test.py + fi + conda deactivate conda env remove -n ${ENV_NAME} fi diff --git a/test/smoke_test/smoke_test.py b/test/smoke_test/smoke_test.py index 226559914..3ae4d9421 100644 --- a/test/smoke_test/smoke_test.py +++ b/test/smoke_test/smoke_test.py @@ -16,6 +16,7 @@ channel = os.getenv("MATRIX_CHANNEL") stable_version = os.getenv("MATRIX_STABLE_VERSION") package_type = os.getenv("MATRIX_PACKAGE_TYPE") +target_os = os.getenv("TARGET_OS") is_cuda_system = gpu_arch_type == "cuda" NIGHTLY_ALLOWED_DELTA = 3 @@ -24,14 +25,14 @@ { "name": "torchvision", "repo": "https://github.com/pytorch/vision.git", - "smoke_test": "python3 ./vision/test/smoke_test.py", + "smoke_test": "./vision/test/smoke_test.py", "extension": "extension", "repo_name": "vision", }, { "name": "torchaudio", "repo": "https://github.com/pytorch/audio.git", - "smoke_test": "python3 ./audio/test/smoke_test/smoke_test.py --no-ffmpeg", + "smoke_test": "./audio/test/smoke_test/smoke_test.py --no-ffmpeg", "extension": "_extension", "repo_name": "audio", }, @@ -212,8 +213,11 @@ def smoke_test_modules(): print(f"Path does not exist: {cwd}/{module['repo_name']}") subprocess.check_output(f"git clone --depth 1 {module['repo']}", stderr=subprocess.STDOUT, shell=True) try: + smoke_test_command = f"python3 {module['smoke_test']}" + if target_os == 'windows': + smoke_test_command = f"python {module['smoke_test']}" output = subprocess.check_output( - module["smoke_test"], stderr=subprocess.STDOUT, shell=True, + smoke_test_command, stderr=subprocess.STDOUT, shell=True, universal_newlines=True) except subprocess.CalledProcessError as exc: raise RuntimeError( From 5ebcc51debca467b64d0cf392fa92e107f8c209c Mon Sep 17 00:00:00 2001 From: atalman Date: Thu, 29 Jun 2023 14:19:38 -0700 Subject: [PATCH 8/8] Uninstall fix --- .github/scripts/validate_binaries.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/validate_binaries.sh b/.github/scripts/validate_binaries.sh index 87a4d96e1..c6a8afffc 100755 --- a/.github/scripts/validate_binaries.sh +++ b/.github/scripts/validate_binaries.sh @@ -9,8 +9,7 @@ else # Make sure we remove previous installation if it exists if [[ ${MATRIX_PACKAGE_TYPE} == 'wheel' ]]; then - UNINSTALL=${INSTALLATION/"install"/"uninstall -y"} - eval $UNINSTALL + pip3 uninstall -y torch torchaudio torchvision fi eval $INSTALLATION