From c47d1093f8430077a8c00e9b92ddb6f4bf526624 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 12:37:04 -0700 Subject: [PATCH 01/10] Run test_custom_ops on Mac --- .github/workflows/pull.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index be48eb181d6..d66deb677cc 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -51,6 +51,9 @@ jobs: bash .ci/scripts/test.sh popd + # Test custom ops + bash examples/custom_ops/test_custom_ops.sh + cmake-build-test-linux: name: cmake-build-test-linux uses: pytorch/test-infra/.github/workflows/linux_job.yml@main From e406bd7a33ed884df98118f569e3db59caed50a9 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 12:43:13 -0700 Subject: [PATCH 02/10] Update path --- .github/workflows/pull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index d66deb677cc..5bc25e3234d 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -49,10 +49,10 @@ jobs: # Build and test Executorch bash .ci/scripts/test.sh - popd # Test custom ops bash examples/custom_ops/test_custom_ops.sh + popd cmake-build-test-linux: name: cmake-build-test-linux From 043a680a2a0566b6bee366d61d93b3d25ff65ddc Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 12:56:46 -0700 Subject: [PATCH 03/10] Create python3 softlink --- .ci/scripts/setup-macos.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index 43978d8bcf5..b46fc7b79c9 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -50,6 +50,18 @@ install_pip_dependencies() { popd } +create_python3_softlink() { + PYTHON=$(which python) + + # This is a workaround because miniconda setup on MacOS doesn't include + # a python3 binary anymore because python is now always python3 + PYTHON3=$(echo "${PYTHON}" | sed "s/python/python3/") + ln -s "${PYTHON}" "${PYTHON3}" + + which python3 +} + install_buck install_conda install_pip_dependencies +create_python3_softlink From 5a934ab7fc742ad437e0b42483e7ca178e569083 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 13:00:19 -0700 Subject: [PATCH 04/10] Debug --- .ci/scripts/setup-macos.sh | 12 ------------ examples/custom_ops/test_custom_ops.sh | 4 ++++ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index b46fc7b79c9..43978d8bcf5 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -50,18 +50,6 @@ install_pip_dependencies() { popd } -create_python3_softlink() { - PYTHON=$(which python) - - # This is a workaround because miniconda setup on MacOS doesn't include - # a python3 binary anymore because python is now always python3 - PYTHON3=$(echo "${PYTHON}" | sed "s/python/python3/") - ln -s "${PYTHON}" "${PYTHON3}" - - which python3 -} - install_buck install_conda install_pip_dependencies -create_python3_softlink diff --git a/examples/custom_ops/test_custom_ops.sh b/examples/custom_ops/test_custom_ops.sh index 77c6e74a860..0c992d9886d 100644 --- a/examples/custom_ops/test_custom_ops.sh +++ b/examples/custom_ops/test_custom_ops.sh @@ -66,6 +66,10 @@ test_cmake_custom_op_2() { cmake-out/executor_runner "--model_path=./${model_name}.pte" } +# DEBUG +which python3 +python3 -mpip list + test_buck2_custom_op_1 test_cmake_custom_op_1 test_cmake_custom_op_2 From c7a00b6104ffdf3b55b929e29b68d01bd11fb30e Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 13:14:51 -0700 Subject: [PATCH 05/10] Debug --- .ci/scripts/setup-macos.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index 43978d8bcf5..f45fb70bcb1 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -40,6 +40,12 @@ install_conda() { popd } +use_conda_python() { + PYTHON_PATH=$(dirname $(conda run --no-capture-output which python)) + # Use conda python + export PATH="${PYTHON_PATH}:${PATH}" +} + install_pip_dependencies() { pushd .ci/docker # Install all Python dependencies, including PyTorch @@ -52,4 +58,5 @@ install_pip_dependencies() { install_buck install_conda +use_conda_python install_pip_dependencies From 4b201fbc337ecbbd3d0c99f4863273f2fee604e4 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 13:20:50 -0700 Subject: [PATCH 06/10] Debug --- .ci/scripts/setup-macos.sh | 7 ------- .github/workflows/pull.yml | 9 +++++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index f45fb70bcb1..43978d8bcf5 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -40,12 +40,6 @@ install_conda() { popd } -use_conda_python() { - PYTHON_PATH=$(dirname $(conda run --no-capture-output which python)) - # Use conda python - export PATH="${PYTHON_PATH}:${PATH}" -} - install_pip_dependencies() { pushd .ci/docker # Install all Python dependencies, including PyTorch @@ -58,5 +52,4 @@ install_pip_dependencies() { install_buck install_conda -use_conda_python install_pip_dependencies diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 5bc25e3234d..463d67b56aa 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -47,6 +47,15 @@ jobs: # Setup MacOS dependencies as there is no Docker support on MacOS atm bash .ci/scripts/setup-macos.sh + # TODO: This is a workaround because python3 binary isn't in the path, + # so subsequent commands using python3 wrongly uses system python3 instead + PYTHON_PATH=$(dirname $(conda run --no-capture-output which python3)) + # Use conda python3 + export PATH="${PYTHON_PATH}:${PATH}" + + # DEBUG + which python3 + # Build and test Executorch bash .ci/scripts/test.sh From 2e1278a676e8057a22731c87522a286d45f2d9b8 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 13:25:27 -0700 Subject: [PATCH 07/10] Debug --- .github/workflows/pull.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 463d67b56aa..00cdbe8f6ae 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -47,14 +47,15 @@ jobs: # Setup MacOS dependencies as there is no Docker support on MacOS atm bash .ci/scripts/setup-macos.sh + set -exu # TODO: This is a workaround because python3 binary isn't in the path, # so subsequent commands using python3 wrongly uses system python3 instead PYTHON_PATH=$(dirname $(conda run --no-capture-output which python3)) # Use conda python3 export PATH="${PYTHON_PATH}:${PATH}" - # DEBUG which python3 + echo "DONE" # Build and test Executorch bash .ci/scripts/test.sh From 1a8d92ddc27b0473dc46ef1b75a831bbfce2ae32 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 13:31:26 -0700 Subject: [PATCH 08/10] Debug --- .github/workflows/pull.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 00cdbe8f6ae..bfabf9fe5d0 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -50,8 +50,7 @@ jobs: set -exu # TODO: This is a workaround because python3 binary isn't in the path, # so subsequent commands using python3 wrongly uses system python3 instead - PYTHON_PATH=$(dirname $(conda run --no-capture-output which python3)) - # Use conda python3 + PYTHON_PATH=$(dirname $(conda run --no-capture-output which python)) export PATH="${PYTHON_PATH}:${PATH}" # DEBUG which python3 From a8ceda2a3b60346a3c47d5ef731124184903ab0b Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 14:24:55 -0700 Subject: [PATCH 09/10] Add macos_kernel_link_options --- .github/workflows/pull.yml | 6 ++---- CMakeLists.txt | 6 +++++- build/Utils.cmake | 8 ++++++++ examples/custom_ops/CMakeLists.txt | 6 +++++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index bfabf9fe5d0..25c1f518df3 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -49,12 +49,10 @@ jobs: set -exu # TODO: This is a workaround because python3 binary isn't in the path, - # so subsequent commands using python3 wrongly uses system python3 instead + # so subsequent commands wrongly used system python3, which didn't have + # executorch PYTHON_PATH=$(dirname $(conda run --no-capture-output which python)) export PATH="${PYTHON_PATH}:${PATH}" - # DEBUG - which python3 - echo "DONE" # Build and test Executorch bash .ci/scripts/test.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 47c28d7e506..23c980e7160 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -259,7 +259,11 @@ target_link_libraries(portable_kernels_bindings INTERFACE portable_kernels) # Ensure that the load-time constructor functions run. By default, the linker # would remove them since there are no other references to them. -kernel_link_options(portable_kernels_bindings) +if(APPLE) + macos_kernel_link_options(portable_kernels_bindings) +else() + kernel_link_options(portable_kernels_bindings) +endif() # # executor_runner: A simple commandline tool that loads and runs a program file. diff --git a/build/Utils.cmake b/build/Utils.cmake index fdaf6ab77f0..8ed11fc1efb 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -32,3 +32,11 @@ function(kernel_link_options target_name) -Wl,--no-whole-archive ) endfunction() + +function(macos_kernel_link_options target_name) + target_link_options(${target_name} + INTERFACE + # Same as kernel_link_options but it's for MacOS linker + -Wl,-force_load,$ + ) +endfunction() diff --git a/examples/custom_ops/CMakeLists.txt b/examples/custom_ops/CMakeLists.txt index a4876d7ac28..5660637ca30 100644 --- a/examples/custom_ops/CMakeLists.txt +++ b/examples/custom_ops/CMakeLists.txt @@ -110,7 +110,11 @@ if(REGISTER_EXAMPLE_CUSTOM_OP_2) # Ensure that the load-time constructor functions run. By default, the linker # would remove them since there are no other references to them. - kernel_link_options("custom_ops_aot_lib") + if(APPLE) + macos_kernel_link_options("custom_ops_aot_lib") + else() + kernel_link_options("custom_ops_aot_lib") + endif() endif() # 1. C++ library to register custom ops into Executorch runtime. From b2dfc0bbb63baf1fc0aaa3c85d9369b10cf7ec7d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 16 Aug 2023 14:29:23 -0700 Subject: [PATCH 10/10] Add macos_kernel_link_options --- .github/workflows/pull.yml | 10 ---------- examples/custom_ops/test_custom_ops.sh | 4 ---- 2 files changed, 14 deletions(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 25c1f518df3..be48eb181d6 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -47,18 +47,8 @@ jobs: # Setup MacOS dependencies as there is no Docker support on MacOS atm bash .ci/scripts/setup-macos.sh - set -exu - # TODO: This is a workaround because python3 binary isn't in the path, - # so subsequent commands wrongly used system python3, which didn't have - # executorch - PYTHON_PATH=$(dirname $(conda run --no-capture-output which python)) - export PATH="${PYTHON_PATH}:${PATH}" - # Build and test Executorch bash .ci/scripts/test.sh - - # Test custom ops - bash examples/custom_ops/test_custom_ops.sh popd cmake-build-test-linux: diff --git a/examples/custom_ops/test_custom_ops.sh b/examples/custom_ops/test_custom_ops.sh index 0c992d9886d..77c6e74a860 100644 --- a/examples/custom_ops/test_custom_ops.sh +++ b/examples/custom_ops/test_custom_ops.sh @@ -66,10 +66,6 @@ test_cmake_custom_op_2() { cmake-out/executor_runner "--model_path=./${model_name}.pte" } -# DEBUG -which python3 -python3 -mpip list - test_buck2_custom_op_1 test_cmake_custom_op_1 test_cmake_custom_op_2