From f216df6ff6d33becc7b9d2c2dad535880ea12d36 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Thu, 8 Sep 2022 20:12:03 -0500 Subject: [PATCH 1/4] Adding onetrace_enabled Example of usage: ``` (dev_dpctl) opavlyk@opavlyk-mobl:~/repos/dpctl$ ~/pti-gpu/tools/onetrace/build/onetrace --conditional-collection -v -t --demangle ipython Device Timeline: start time (CLOCK_MONOTONIC_RAW) [ns] = 207284739877222 Device Timeline: start time (CLOCK_MONOTONIC) [ns] = 207284790110055 Device Timeline: start time (CLOCK_REALTIME) [ns] = 1662686020459161393 Python 3.9.12 (main, Jun 1 2022, 11:38:51) Type 'copyright', 'credits' or 'license' for more information IPython 8.4.0 -- An enhanced Interactive Python. Type '?' for help. In [1]: import dpctl.tensor as dpt, dpctl, numpy as np Device Timeline: start time (CLOCK_MONOTONIC_RAW) [ns] = 207294358012969 Device Timeline: start time (CLOCK_MONOTONIC) [ns] = 207294408245381 Device Timeline: start time (CLOCK_REALTIME) [ns] = 1662686030077296715 Device Timeline: start time (CLOCK_MONOTONIC_RAW) [ns] = 207294447416361 Device Timeline: start time (CLOCK_MONOTONIC) [ns] = 207294497648642 Device Timeline: start time (CLOCK_REALTIME) [ns] = 1662686030166699977 Device Timeline: start time (CLOCK_MONOTONIC_RAW) [ns] = 207294522074685 Device Timeline: start time (CLOCK_MONOTONIC) [ns] = 207294572308151 Device Timeline: start time (CLOCK_REALTIME) [ns] = 1662686030241359487 Device Timeline: start time (CLOCK_MONOTONIC_RAW) [ns] = 207294602756763 Device Timeline: start time (CLOCK_MONOTONIC) [ns] = 207294652988740 Device Timeline: start time (CLOCK_REALTIME) [ns] = 1662686030322040075 In [2]: from dpctl.utils import onetrace_enabled In [3]: with onetrace_enabled(): ...: dpt.linspace(0, 1, num=1000, dtype='f4') ...: dpt.arange(0, 20, dtype='i4') ...: Device Timeline (queue: 0x56315e00e720): linear_sequence_affine_kernel[SIMD32 {5; 1; 1} {200; 1; 1}]<1.1> [ns] = 49099144803 (append) 49099446053 (submit) 49101192928 (start) 49101202303 (end) Device Timeline (queue: 0x56315e00e720): linear_sequence_step_kernel[SIMD32 {1; 1; 1} {20; 1; 1}]<2.1> [ns] = 49154332484 (append) 49154668024 (submit) 49155196149 (start) 49155213440 (end) In [4]: quit Segmentation fault ``` The segmenation fault at the exit is a known issue with ZE-tracer module. --- dpctl/utils/__init__.py | 2 ++ dpctl/utils/_onetrace_context.py | 53 ++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 dpctl/utils/_onetrace_context.py diff --git a/dpctl/utils/__init__.py b/dpctl/utils/__init__.py index 5601290af6..7589f9de9f 100644 --- a/dpctl/utils/__init__.py +++ b/dpctl/utils/__init__.py @@ -23,9 +23,11 @@ get_execution_queue, validate_usm_type, ) +from ._onetrace_context import onetrace_enabled __all__ = [ "get_execution_queue", "get_coerced_usm_type", "validate_usm_type", + "onetrace_enabled", ] diff --git a/dpctl/utils/_onetrace_context.py b/dpctl/utils/_onetrace_context.py new file mode 100644 index 0000000000..87e9ea582c --- /dev/null +++ b/dpctl/utils/_onetrace_context.py @@ -0,0 +1,53 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2020-2022 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from contextlib import contextmanager + + +@contextmanager +def onetrace_enabled(): + """ Enable `onetrace` collection for kernels executed in this context. + + N.B.: Proper working of this utility assumes that Python interpreter + has been launched by `onetrace` tool from intel/pti-gpu project. + + .. Example:: bash + + $ onetrace --conditional-collection -v -t --demangle \ + python script.py + + .. Example:: bash + + import dpctl.tensor as dpt + from dpctl.utils import onetrace_enabled + + # onetrace output reporting on execution of the kernel + # should be seen, starting with "Device Timeline" + with onetrace_enabled(): + dpt.arange(100, dtype='int16') + + """ + _env_var_name = "PTI_ENABLE_COLLECTION" + saved = os.getenv(_env_var_name, None) + try: + os.environ[_env_var_name] = "1" + yield + finally: + if saved is None: + del os.environ[_env_var_name] + else: + os.environ[_env_var_name] = saved From 22f2df5f2fb0436c0176db732399d03f3766ea93 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Fri, 9 Sep 2022 06:49:07 -0500 Subject: [PATCH 2/4] Added test for onetrace_enabled() context manager. Fixed docstring example --- dpctl/tests/test_utils.py | 10 ++++++++++ dpctl/utils/_onetrace_context.py | 11 ++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/dpctl/tests/test_utils.py b/dpctl/tests/test_utils.py index 5960d3fbe5..a8e331da03 100644 --- a/dpctl/tests/test_utils.py +++ b/dpctl/tests/test_utils.py @@ -111,3 +111,13 @@ def validate_usm_type_arg(): dpctl.utils.validate_usm_type("inv", allow_none=True) with pytest.raises(ValueError): dpctl.utils.validate_usm_type("inv", allow_none=False) + + +def test_onetrace_enabled(): + import os + + v_name = "PTI_ENABLE_COLLECTION" + v_v = os.getenv(v_name, None) + with dpctl.utils.onetrace_enabled(): + assert os.getenv(v_name, None) == "1" + assert os.getenv(v_name, None) == v_v diff --git a/dpctl/utils/_onetrace_context.py b/dpctl/utils/_onetrace_context.py index 87e9ea582c..a690d147b2 100644 --- a/dpctl/utils/_onetrace_context.py +++ b/dpctl/utils/_onetrace_context.py @@ -20,17 +20,18 @@ @contextmanager def onetrace_enabled(): - """ Enable `onetrace` collection for kernels executed in this context. + """Enable `onetrace` collection for kernels executed in this context. N.B.: Proper working of this utility assumes that Python interpreter has been launched by `onetrace` tool from intel/pti-gpu project. - .. Example:: bash + :Example: + Launch the Python interpreter using `onetrace` tool: :: - $ onetrace --conditional-collection -v -t --demangle \ - python script.py + $ onetrace --conditional-collection -v -t --demangle python app.py - .. Example:: bash + Now using the context manager in the Python sessions enables + data collection and its output for every offloaded kernel :: import dpctl.tensor as dpt from dpctl.utils import onetrace_enabled From c8e74e90aa74cca505a99dc0ff348bcd72294d76 Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 11 Sep 2022 08:32:49 -0500 Subject: [PATCH 3/4] Add detection to warn user of context is used by onetrace was not onetrace tool on linux uses execvp to start the application to trace. It sets LD_PRELOAD and PTI_ENABLE environment variables https://github.com/intel/pti-gpu/blob/master/loader/loader.cc#L216-L217 The check is added, on Linux, to confirm once that environment does contain expected variables, and emit a warning otherwise. --- dpctl/utils/_onetrace_context.py | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/dpctl/utils/_onetrace_context.py b/dpctl/utils/_onetrace_context.py index a690d147b2..b7e68e69bf 100644 --- a/dpctl/utils/_onetrace_context.py +++ b/dpctl/utils/_onetrace_context.py @@ -14,8 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os from contextlib import contextmanager +from os import environ, getenv +from platform import system as sys_platform + +_UNCHECKED = sys_platform() == "Linux" +del sys_platform @contextmanager @@ -42,13 +46,31 @@ def onetrace_enabled(): dpt.arange(100, dtype='int16') """ + global _UNCHECKED + + if _UNCHECKED: + _UNCHECKED = False + if not ( + getenv("PTI_ENABLE", None) == "1" + and "onetrace_tool" in getenv("LD_PRELOAD", "") + ): + import warnings + + warnings.warn( + "It looks like Python interpreter was not started using " + "`onetrace` utility. Using `onetrace_enabled` may have " + "no effect. See `onetrace_enabled.__doc__` for usage.", + RuntimeWarning, + stacklevel=2, + ) + _env_var_name = "PTI_ENABLE_COLLECTION" - saved = os.getenv(_env_var_name, None) + saved = getenv(_env_var_name, None) try: - os.environ[_env_var_name] = "1" + environ[_env_var_name] = "1" yield finally: if saved is None: - del os.environ[_env_var_name] + del environ[_env_var_name] else: - os.environ[_env_var_name] = saved + environ[_env_var_name] = saved From d03f0854cade16b76b79d89474b99071263775fb Mon Sep 17 00:00:00 2001 From: Oleksandr Pavlyk Date: Sun, 11 Sep 2022 08:54:20 -0500 Subject: [PATCH 4/4] Filter possible RuntimeWarning out --- dpctl/tests/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dpctl/tests/test_utils.py b/dpctl/tests/test_utils.py index a8e331da03..df4a9f503f 100644 --- a/dpctl/tests/test_utils.py +++ b/dpctl/tests/test_utils.py @@ -113,6 +113,7 @@ def validate_usm_type_arg(): dpctl.utils.validate_usm_type("inv", allow_none=False) +@pytest.mark.filterwarnings("ignore:.*:RuntimeWarning") def test_onetrace_enabled(): import os