diff --git a/.ci/docker/conda-env-ci.txt b/.ci/docker/conda-env-ci.txt index 467bac5e422..945958a6ed4 100644 --- a/.ci/docker/conda-env-ci.txt +++ b/.ci/docker/conda-env-ci.txt @@ -1,3 +1,4 @@ # TODO: We might need to update this to install flatbuffers from the pinned commit # in fbcode/executorch/third-party/flatbuffers.submodule.txt flatbuffers=2.0.0 +cmake=3.22.1 diff --git a/.ci/docker/requirements-ci.txt b/.ci/docker/requirements-ci.txt index 88793b9498c..daea5048832 100644 --- a/.ci/docker/requirements-ci.txt +++ b/.ci/docker/requirements-ci.txt @@ -3,4 +3,5 @@ numpy==1.25.2 PyYAML==6.0.1 ruamel.yaml==0.17.32 sympy==1.12 +tomli==2.0.1 zstd==1.5.5.1 diff --git a/.ci/scripts/test-cmake.sh b/.ci/scripts/test-cmake.sh new file mode 100755 index 00000000000..975e7aa016e --- /dev/null +++ b/.ci/scripts/test-cmake.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -exu + +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" + +test_model() { + MODEL_NAME=$1 + python -m examples.export.export_example --model_name="${MODEL_NAME}" + + # Run test model + ./"${CMAKE_OUTPUT_DIR}"/executor_runner --model_path "./${MODEL_NAME}.pte" +} + +build_and_test_executorch() { + CMAKE_OUTPUT_DIR=cmake-out + # Build executorch runtime using cmake + rm -rf "${CMAKE_OUTPUT_DIR}" && mkdir "${CMAKE_OUTPUT_DIR}" + + pushd "${CMAKE_OUTPUT_DIR}" + cmake -DBUCK2=buck2 .. + popd + + if [ "$(uname)" == "Darwin" ]; then + CMAKE_JOBS=$(( $(sysctl -n hw.ncpu) - 1 )) + else + CMAKE_JOBS=$(( $(nproc) - 1 )) + fi + cmake --build "${CMAKE_OUTPUT_DIR}" -j "${CMAKE_JOBS}" + + which python + # Test the example linear model + test_model "linear" +} + +install_executorch +build_and_test_executorch diff --git a/.ci/scripts/test.sh b/.ci/scripts/test.sh index 89f274e1962..53569342c56 100755 --- a/.ci/scripts/test.sh +++ b/.ci/scripts/test.sh @@ -7,13 +7,15 @@ set -exu -install_executorch() { - which pip - # Install executorch, this assumes that Executorch is checked out in the - # current directory - pip install . - # Just print out the list of packages for debugging - pip list +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" + +test_model() { + MODEL_NAME=$1 + python -m examples.export.export_example --model_name="${MODEL_NAME}" + + # Run test model + buck2 run //examples/executor_runner:executor_runner -- --model_path "./${MODEL_NAME}.pte" } build_and_test_executorch() { @@ -21,10 +23,8 @@ build_and_test_executorch() { buck2 build //examples/executor_runner:executor_runner which python - # Export a test model - python -m examples.export.export_example --model_name="linear" - # Run test model - buck2 run //examples/executor_runner:executor_runner -- --model_path ./linear.pte + # Test the example linear model + test_model "linear" } install_executorch diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh new file mode 100644 index 00000000000..a14f16a6170 --- /dev/null +++ b/.ci/scripts/utils.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +install_executorch() { + which pip + # Install executorch, this assumes that Executorch is checked out in the + # current directory + pip install . + # Just print out the list of packages for debugging + pip list +} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index fc355e59b1e..be48eb181d6 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -30,6 +30,9 @@ jobs: # Build and test Executorch bash .ci/scripts/test.sh + # Test custom ops + bash examples/custom_ops/test_custom_ops.sh + buck-build-test-macos: name: buck-build-test-macos uses: pytorch/test-infra/.github/workflows/macos_job.yml@main @@ -47,3 +50,21 @@ jobs: # Build and test Executorch bash .ci/scripts/test.sh popd + + cmake-build-test-linux: + name: cmake-build-test-linux + uses: pytorch/test-infra/.github/workflows/linux_job.yml@main + with: + runner: linux.2xlarge + docker-image: executorch-ubuntu-22.04-clang12 + fetch-depth: 0 + submodules: 'true' + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} + script: | + PYTHON_VERSION=3.10 + # TODO: Figure out why /opt/conda/envs/py_$PYTHON_VERSION/bin is not in the path + # here, as it's there in the container + export PATH="/opt/conda/envs/py_${PYTHON_VERSION}/bin:${PATH}" + + # Build and test Executorch + bash .ci/scripts/test-cmake.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index cdf2dbe8be3..47c28d7e506 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -259,19 +259,7 @@ target_link_libraries(portable_kernels_bindings INTERFACE portable_kernels) # Ensure that the load-time constructor functions run. By default, the linker # would remove them since there are no other references to them. -if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL - "AppleClang") - if(CMAKE_TOOLCHAIN_FILE MATCHES ".*android\.toolchain\.cmake$") - # For Android tool chain - gcc_kernel_link_options(portable_kernels_bindings) - else() - # For host tool chain - clang_kernel_link_options(portable_kernels_bindings) - endif() -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # Using gcc - gcc_kernel_link_options(portable_kernels_bindings) -endif() +kernel_link_options(portable_kernels_bindings) # # executor_runner: A simple commandline tool that loads and runs a program file. diff --git a/build/Utils.cmake b/build/Utils.cmake index 79533929637..fdaf6ab77f0 100644 --- a/build/Utils.cmake +++ b/build/Utils.cmake @@ -22,20 +22,8 @@ function(executorch_print_configuration_summary) message(STATUS " REGISTER_EXAMPLE_CUSTOM_OPS : ${REGISTER_EXAMPLE_CUSTOM_OPS}") endfunction() -# This is the funtion to use -Wl to link static library, used for clang -function(clang_kernel_link_options target_name) - target_link_options(${target_name} - INTERFACE - # TODO(dbort): This will cause the .a to show up on the link line twice - # for targets that depend on this library; once because CMake will add - # it, and once because it's in this list of args. See if there's a way - # to avoid that. - -Wl,-force_load,$ - ) -endfunction() - -# This is the funtion to use -Wl, --whole-archive to link static library, used for gcc -function(gcc_kernel_link_options target_name) +# This is the funtion to use -Wl, --whole-archive to link static library +function(kernel_link_options target_name) target_link_options(${target_name} INTERFACE # TODO(dbort): This will cause the .a to show up on the link line twice diff --git a/examples/custom_ops/CMakeLists.txt b/examples/custom_ops/CMakeLists.txt index cce3b23b19d..a4876d7ac28 100644 --- a/examples/custom_ops/CMakeLists.txt +++ b/examples/custom_ops/CMakeLists.txt @@ -31,6 +31,9 @@ endif() if(NOT TORCH_ROOT) set(TORCH_ROOT ${EXECUTORCH_ROOT}/third-party/pytorch) endif() + +include(${EXECUTORCH_ROOT}/build/Utils.cmake) + # Command to generate selected_operators.yaml from custom_ops.yaml. set(_oplist_yaml ${CMAKE_CURRENT_BINARY_DIR}/selected_operators.yaml) file(GLOB_RECURSE _codegen_tools_srcs "${EXECUTORCH_ROOT}/codegen/tools/*.py") @@ -107,16 +110,7 @@ if(REGISTER_EXAMPLE_CUSTOM_OP_2) # Ensure that the load-time constructor functions run. By default, the linker # would remove them since there are no other references to them. - if((CMAKE_CXX_COMPILER_ID MATCHES "AppleClang") - OR (APPLE AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")) - target_link_options(custom_ops_aot_lib INTERFACE - "-Wl,-force_load,$") - elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") - target_link_options( - custom_ops_aot_lib INTERFACE - "-Wl,--whole-archive,$,--no-whole-archive" - ) - endif() + kernel_link_options("custom_ops_aot_lib") endif() # 1. C++ library to register custom ops into Executorch runtime.