[EXAMPLE] Example of calling MKL on usm_ndarray inputs (#780)

* Example of calling MKL on usm_ndarray inputs

* gemv function returns a pair of events, first for host task, second for MKL computation

* test_example job needs scikit-build

Author: oleksandr-pavlyk

.github/workflows/conda-package.yml

@@ -317,7 +317,7 @@ jobs:
     runs-on:  ${{ matrix.runner }}
     strategy:
       matrix:
-        python: [3.8]
+        python: [3.9]
         experimental: [false]
         runner: [ubuntu-latest]
     continue-on-error: ${{ matrix.experimental }}
@@ -363,57 +363,79 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-
             ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-
-      - name: Install dpctl
+      - name: Install example requirements
         shell: bash -l {0}
         run: |
+          CHANNELS="${{ env.CHANNELS }}"
           source $CONDA/etc/profile.d/conda.sh
-          conda activate
-          CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}"
-          conda install -y $PACKAGE_NAME pytest python=${{ matrix.python }} $CHANNELS
-          # Test installed packages
-          conda list
-      - name: Install example requirements
+          conda create -n examples -y pytest python=${{ matrix.python }} $CHANNELS
+          conda install -n examples -y cmake">=3.22" $CHANNELS || exit 1
+          conda install -n examples -y ninja $CHANNELS || exit 1
+          conda install -n examples -y pybind11 cython scikit-build $CHANNELS || exit 1
+          conda install -n examples -y mkl-dpcpp mkl-devel-dpcpp dpcpp_cpp_rt $CHANNELS || exit 1
+          conda create -y -n build_env -c intel dpcpp_linux-64
+      - name: Install dpctl
         shell: bash -l {0}
         run: |
           source $CONDA/etc/profile.d/conda.sh
-          conda install -y pybind11 cython
-          conda install -y -c intel mkl-dpcpp mkl-devel-dpcpp numba-dppy
-          conda create -y -n build_env -c intel dpcpp_linux-64
+          conda activate
+          CHANNELS="-c $GITHUB_WORKSPACE/channel -c dppy/label/dev -c intel --override-channels"
+          conda install -n examples -y $CHANNELS numba-dppy numpy dpctl || exit 1
       - name: Build and run examples with native extensions
         shell: bash -l {0}
         run: |
           source $CONDA/etc/profile.d/conda.sh
           export OCL_ICD_FILENAMES=libintelocl.so
           export SYCL_ENABLE_HOST_DEVICE=1
-          conda activate
+          conda activate examples
+          conda list
           cd examples/pybind11
-          export CC=dpcpp
-          export CXX=dpcpp
           for d in $(ls)
           do
             pushd $d
+            export MKLROOT=${CONDA_PREFIX}
+            export TBBROOT=${CONDA_PREFIX}
             conda activate --stack build_env
-            python setup.py build_ext --inplace || exit 1
+            if [ -e CMakeLists.txt ]
+            then
+               python setup.py build_ext --inplace -- -G Ninja \
+                    -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx \
+                    -DTBB_LIBRARY_DIR=${TBBROOT}/lib \
+                    -DMKL_LIBRARY_DIR=${MKLROOT}/lib \
+                    -DMKL_INCLUDE_DIR=${MKLROOT}/include \
+                    -DTBB_INCLUDE_DIR=${TBBROOT}/include || exit 1
+            else
+               CC=dpcpp CXX=dpcpp LD_SHARED="dpcpp -shared" \
+                    python setup.py build_ext --inplace || exit 1
+            fi
             conda deactivate
-            python example.py
+            if [ -e tests ]
+            then
+               LD_LIBRARY_PATH=${CONDA_PREFIX}/lib python -m pytest tests || exit 1
+            else
+               LD_LIBRARY_PATH=${CONDA_PREFIX}/lib python example.py || exit 1
+            fi
             popd
           done
           cd ../cython
           for d in $(ls)
           do
             pushd $d
             conda activate --stack build_env
-            python setup.py build_ext --inplace || exit 1
+            CC=dpcpp CXX=dpcpp LD_SHARED="dpcpp -shared" \
+                   python setup.py build_ext --inplace || exit 1
             conda deactivate
-            python run.py
+            LD_LIBRARY_PATH=${CONDA_PREFIX}/lib python run.py || exit 1
             popd
           done
       - name: Run Python examples
         shell: bash -l {0}
         run: |
           cd examples/python
+          source $CONDA/etc/profile.d/conda.sh
           export OCL_ICD_FILENAMES=libintelocl.so
           export SYCL_ENABLE_HOST_DEVICE=1
+          conda activate examples
           for script in $(find . \( -not -name "_*" -and -name "*.py" \))
           do
             echo "Executing ${script}"

cmake/FindDpctl.cmake

@@ -0,0 +1,64 @@
+#.rst:
+#
+# Find the include directory for ``dpctl_capi.h``, ``dpctl4pybind11.hpp``.
+#
+# This module sets the following variables:
+#
+# ``Dpctl_FOUND``
+#   True if DPCTL was found.
+# ``Dpctl_INCLUDE_DIRS``
+#   The include directories needed to use Dpctl.
+# ``Dpctl_VERSION``
+#   The version of DPCTL found.
+#
+# The module will also explicitly define one cache variable:
+#
+# ``Dpctl_INCLUDE_DIR``
+#
+
+if(NOT Dpctl_FOUND)
+  set(_find_extra_args)
+  if(Dpctl_FIND_REQUIRED)
+    list(APPEND _find_extra_args REQUIRED)
+  endif()
+  if(Dpctl_FIND_QUIET)
+    list(APPEND _find_extra_args QUIET)
+  endif()
+  find_package(PythonInterp ${_find_extra_args})
+  find_package(PythonLibs ${_find_extra_args})
+
+  if(PYTHON_EXECUTABLE)
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}"
+      -c "import dpctl; print(dpctl.get_include())"
+      OUTPUT_VARIABLE _dpctl_include_dir
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      ERROR_QUIET
+      )
+    execute_process(COMMAND "${PYTHON_EXECUTABLE}"
+      -c "import dpctl; print(dpctl.__version__)"
+      OUTPUT_VARIABLE Dpctl_VERSION
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      ERROR_QUIET
+      )
+
+  endif()
+endif()
+
+find_path(Dpctl_INCLUDE_DIR
+  dpctl_capi.h dpctl4pybind11.hpp dpctl_sycl_interface.h
+  PATHS "${_dpctl_include_dir}" "${PYTHON_INCLUDE_DIR}"
+  PATH_SUFFIXES dpctl/include
+  )
+
+set(Dpctl_INCLUDE_DIRS ${Dpctl_INCLUDE_DIR})
+
+# handle the QUIETLY and REQUIRED arguments and set Dpctl_FOUND to TRUE if
+# all listed variables are TRUE
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Dpctl
+                                  REQUIRED_VARS
+                                    Dpctl_INCLUDE_DIR
+                                  VERSION_VAR Dpctl_VERSION
+                                  )
+
+mark_as_advanced(Dpctl_INCLUDE_DIR)

examples/pybind11/onemkl_gemv/CMakeLists.txt

@@ -0,0 +1,57 @@
+cmake_minimum_required(VERSION 3.22 FATAL_ERROR)
+
+project(example_use_mkl_gemm LANGUAGES CXX)
+set(DPCTL_CMAKE_MODULES_PATH "${CMAKE_SOURCE_DIR}/../../../cmake")
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${DPCTL_CMAKE_MODULES_PATH})
+find_package(IntelDPCPP REQUIRED PATHS ${DPCTL_CMAKE_MODULES_PATH} NO_DEFAULT_PATH)
+
+
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}")
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+# Define CMAKE_INSTALL_xxx: LIBDIR, INCLUDEDIR
+include(GNUInstallDirs)
+
+# Fetch pybind11
+include(FetchContent)
+FetchContent_Declare(
+  pybind11
+  URL https://github.com/pybind/pybind11/archive/refs/tags/v2.9.0.tar.gz
+  URL_HASH SHA256=057fb68dafd972bc13afb855f3b0d8cf0fa1a78ef053e815d9af79be7ff567cb
+)
+FetchContent_MakeAvailable(pybind11)
+
+find_package(PythonExtensions REQUIRED)
+find_package(Dpctl REQUIRED)
+
+find_library(mkl_core NAMES mkl_core PATH ${MKL_LIBRARY_DIR})
+find_library(mkl_sycl NAMES mkl_sycl PATH ${MKL_LIBRARY_DIR})
+find_library(mkl_intel_ilp64 NAMES mkl_intel_ilp64 PATH ${MKL_LIBRARY_DIR})
+find_library(mkl_tbb_thread NAMES mkl_tbb_thread PATH ${MKL_LIBRARY_DIR})
+find_library(tbb NAMES tbb PATH ${TBB_LIBRARY_DIR})
+
+set(py_module_name _onemkl)
+
+pybind11_add_module(${py_module_name}
+    MODULE
+    sycl_gemm/_onemkl.cpp
+)
+target_include_directories(${py_module_name}
+    PUBLIC ${MKL_INCLUDE_DIR} ${TBB_INCLUDE_DIR}
+)
+target_link_libraries(${py_module_name}
+    PUBLIC mkl_sycl mkl_intel_ilp64 mkl_tbb_thread mkl_core tbb
+)
+
+install(TARGETS ${py_module_name} DESTINATION sycl_gemm)
+target_include_directories(${py_module_name} PUBLIC ${Dpctl_INCLUDE_DIRS})
+target_link_directories(${py_module_name} PUBLIC ${MKL_LIBRARY_DIR} ${TBB_LIBRARY_DIR})
+
+get_target_property(_sycl_gemm_sources ${py_module_name} SOURCES)
+set_source_files_properties(${_sycl_gemm_sources}
+  PROPERTIES
+  COMPILE_OPTIONS "-O3;-Wno-deprecated-declarations"
+)
+
+set(ignoreMe "${SKBUILD}")

examples/pybind11/onemkl_gemv/README.md

@@ -0,0 +1,13 @@
+Example of SYCL built pybind11 extension
+
+To build, use (assumes scikit-build and dpcpp) is installed
+
+```sh
+python setup.py develop -- -G "Ninja" -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DTBB_LIBRARY_DIR=$CONDA_PREFIX/lib -DMKL_LIBRARY_DIR=${CONDA_PREFIX}/lib -DMKL_INCLUDE_DIR=${CONDA_PREFIX}/include -DTBB_INCLUDE_DIR=${CONDA_PREFIX}/include
+```
+
+To run test suite
+
+```sh
+python -m pytest tests
+```

examples/pybind11/onemkl_gemv/setup.py

@@ -0,0 +1,10 @@
+from skbuild import setup
+
+setup(
+    name="sycl_gemm",
+    version="0.0.1",
+    description="an example of SYCL-powered Python package (with pybind11)",
+    author="Intel Scripting",
+    license="Apache 2.0",
+    packages=["sycl_gemm"],
+)

examples/pybind11/onemkl_gemv/sycl_gemm/__init__.py

@@ -0,0 +1,3 @@
+from ._onemkl import gemv
+
+__all__ = ["gemv"]