From 40aa1b29e7238fd151c0d66cf437c2dd30677856 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Wed, 7 Jul 2021 17:18:16 -0700 Subject: [PATCH 01/11] [LIT][L0] test for multi-context (c-slice) support in level_zero. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 231 ++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 SYCL/Basic/subsubdevice.cpp diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp new file mode 100644 index 0000000000..35bf672b5f --- /dev/null +++ b/SYCL/Basic/subsubdevice.cpp @@ -0,0 +1,231 @@ +// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out +// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE +// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// +// Intel OpenCL CPU Runtime supports device partition on all (multi-core) +// platforms. Other devices may not support this. + +#include +#include +#include + +using namespace cl::sycl; + +// Log to the same stream as SYCL_PI_TRACE +static void log_pi(const char *msg) { std::cout << msg << std::endl; } + +static void use_mem(buffer buf, queue q) { + q.submit([&](handler &cgh) { + auto acc = buf.get_access(cgh); + cgh.parallel_for(range<1>(buf.size()), + [=](item<1> itemID) { acc[itemID] += 1; }); + }); + q.wait(); +} + +typedef std::vector (*partition_fn)(device dev); + +static std::vector partition_equally(device dev) { + std::vector subdevices = + dev.create_sub_devices(1); + + return subdevices; +} + +static bool check_separate(device dev, buffer buf, + partition_fn partition) { + log_pi("Create sub devices"); + std::vector subdevices = partition(dev); + assert(subdevices.size() > 1); + // CHECK-SEPARATE: Create sub devices + // CHECK-SEPARATE: ---> piDevicePartition + + log_pi("Create sub sub devices of sub-device 0"); + std::vector subsubdevices = partition(subdevices[0]); + assert(subsubdevices.size() > 1); + // CHECK-SEPARATE: Create sub sub devices of sub-device 0 + // CHECK-SEPARATE: ---> piDevicePartition + + + log_pi("Test sub sub device 0"); + { + queue q0(subsubdevices[0]); + use_mem(buf, q0); + } + // CHECK-SEPARATE: Test sub sub device 0 + // CHECK-SEPARATE: ---> piContextCreate + // CHECK-SEPARATE: ---> piQueueCreate + // CHECK-SEPARATE: ---> piMemBufferCreate + // CHECK-SEPARATE: ---> piEnqueueKernelLaunch + // CHECK-SEPARATE: ---> piEventsWait + + log_pi("Test sub sub device 1"); + { + queue q1(subsubdevices[1]); + use_mem(buf, q1); + } + // CHECK-SEPARATE: Test sub sub device 1 + // CHECK-SEPARATE: ---> piContextCreate + // CHECK-SEPARATE: ---> piQueueCreate + // CHECK-SEPARATE: ---> piMemBufferCreate + // + // Verify that we have a memcpy between subdevices in this case + // CHECK-SEPARATE: ---> piEnqueueMemBuffer{{Map|Read}} + // CHECK-SEPARATE: ---> piEnqueueMemBufferWrite + // + // CHECK-SEPARATE: ---> piEnqueueKernelLaunch + // CHECK-SEPARATE: ---> piEventsWait + + return true; +} + +static bool check_shared_context(device dev, buffer buf, + partition_fn partition) { + log_pi("Create sub devices"); + std::vector subdevices = partition(dev); + assert(subdevices.size() > 1); + // CHECK-SHARED: Create sub devices + // CHECK-SHARED: ---> piDevicePartition + + log_pi("Create sub sub devices of sub device 0"); + std::vector subsubdevices = partition(subdevices[0]); + assert(subsubdevices.size() > 1); + // CHECK-SHARED: Create sub sub devices of sub device 0 + // CHECK-SHARED: ---> piDevicePartition + + // Shared context: queues are bound to specific subdevices, but + // memory does not migrate + log_pi("Create shared context"); + context shared_context(subsubdevices); + // CHECK-SHARED: Create shared context + // CHECK-SHARED: ---> piContextCreate + // + // Make sure that a single context is created: see --implicit-check-not above. + + log_pi("Test sub sub device 0"); + { + queue q0(shared_context, subsubdevices[0]); + use_mem(buf, q0); + } + // CHECK-SHARED: Test sub sub device 0 + // CHECK-SHARED: ---> piQueueCreate + // CHECK-SHARED: ---> piMemBufferCreate + // + // Make sure that a single buffer is created (and shared between subdevices): + // see --implicit-check-not above. + // + // CHECK-SHARED: ---> piEnqueueKernelLaunch + // CHECK-SHARED: ---> piEventsWait + + log_pi("Test sub sub device 1"); + { + queue q1(shared_context, subsubdevices[1]); + use_mem(buf, q1); + } + // CHECK-SHARED: Test sub sub device 1 + // CHECK-SHARED: ---> piQueueCreate + // CHECK-SHARED: ---> piEnqueueKernelLaunch + // CHECK-SHARED: ---> piEventsWait + // CHECK-SHARED: ---> piEnqueueMemBufferRead + + return true; +} + +static bool check_fused_context(device dev, buffer buf, + partition_fn partition) { + log_pi("Create sub devices"); + std::vector subdevices = partition(dev); + assert(subdevices.size() > 1); + // CHECK-FUSED: Create sub devices + // CHECK-FUSED: ---> piDevicePartition + + log_pi("Create sub sub devices of sub device 0"); + std::vector subsubdevices = partition(subdevices[0]); + assert(subsubdevices.size() > 1); + // CHECK-FUSED: Create sub sub devices of sub device 0 + // CHECK-FUSED: ---> piDevicePartition + + // Fused context: same as shared context, but also includes the root device + log_pi("Create fused context"); + std::vector devices; + devices.push_back(dev); + devices.push_back(subdevices[0]); + devices.push_back(subdevices[1]); + context fused_context(devices); + // CHECK-FUSED: Create fused context + // CHECK-FUSED: ---> piContextCreate + // + // Make sure that a single context is created: see --implicit-check-not above. + + log_pi("Test root device"); + { + queue q(fused_context, dev); + use_mem(buf, q); + } + // CHECK-FUSED: Test root device + // CHECK-FUSED: ---> piQueueCreate + // CHECK-FUSED: ---> piMemBufferCreate + // + // Make sure that a single buffer is created (and shared between subdevices + // *and* the root device): see --implicit-check-not above. + // + // CHECK-FUSED: ---> piEnqueueKernelLaunch + // CHECK-FUSED: ---> piEventsWait + + log_pi("Test sub sub device 0"); + { + queue q0(fused_context, subsubdevices[0]); + use_mem(buf, q0); + } + // CHECK-FUSED: Test sub sub device 0 + // CHECK-FUSED: ---> piQueueCreate + // CHECK-FUSED: ---> piEnqueueKernelLaunch + // CHECK-FUSED: ---> piEventsWait + + log_pi("Test sub sub device 1"); + { + queue q1(fused_context, subsubdevices[1]); + use_mem(buf, q1); + } + // CHECK-FUSED: Test sub device 1 + // CHECK-FUSED: ---> piQueueCreate + // CHECK-FUSED: ---> piEnqueueKernelLaunch + // CHECK-FUSED: ---> piEventsWait + // CHECK-FUSED: ---> piEnqueueMemBufferRead + + return true; +} + +int main(int argc, const char **argv) { + assert(argc == 3 && "Invalid number of arguments"); + std::string test(argv[1]); + std::string partition_type(argv[2]); + + default_selector selector; + device dev(selector); + + std::vector host_mem(1024, 1); + buffer buf(&host_mem[0], host_mem.size()); + + partition_fn partition; + if (partition_type == "equally") { + partition = partition_equally; + } else { + assert(0 && "Unsupported partition type"); + } + + bool result = false; + if (test == "separate") { + result = check_separate(dev, buf, partition); + } else if (test == "shared") { + result = check_shared_context(dev, buf, partition); + }else { + assert(0 && "Unknown test"); + } + + if (!result) { + fprintf(stderr, "FAILED\n"); + return EXIT_FAILURE; + } +} From b5540d0080ac12d6b048a2d409174ad4a034b8bf Mon Sep 17 00:00:00 2001 From: rehana begam Date: Wed, 7 Jul 2021 17:33:37 -0700 Subject: [PATCH 02/11] clang formatted. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index 35bf672b5f..6d7d8b40ba 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -41,12 +41,11 @@ static bool check_separate(device dev, buffer buf, // CHECK-SEPARATE: Create sub devices // CHECK-SEPARATE: ---> piDevicePartition - log_pi("Create sub sub devices of sub-device 0"); + log_pi("Create sub sub devices of sub device 0"); std::vector subsubdevices = partition(subdevices[0]); assert(subsubdevices.size() > 1); - // CHECK-SEPARATE: Create sub sub devices of sub-device 0 + // CHECK-SEPARATE: Create sub sub devices of sub device 0 // CHECK-SEPARATE: ---> piDevicePartition - log_pi("Test sub sub device 0"); { @@ -220,7 +219,9 @@ int main(int argc, const char **argv) { result = check_separate(dev, buf, partition); } else if (test == "shared") { result = check_shared_context(dev, buf, partition); - }else { + } else if (test == "fused") { + result = check_fused_context(dev, buf, partition); + } else { assert(0 && "Unknown test"); } From d5dc72bc80764fb314eb06ed46ae6e04f30c206c Mon Sep 17 00:00:00 2001 From: rehana begam Date: Thu, 8 Jul 2021 01:10:17 -0700 Subject: [PATCH 03/11] clang formatted and partition fixed. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index 6d7d8b40ba..e9ef434aa2 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -1,7 +1,7 @@ // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out -// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE -// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate -// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE +// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate // // Intel OpenCL CPU Runtime supports device partition on all (multi-core) // platforms. Other devices may not support this. @@ -26,6 +26,15 @@ static void use_mem(buffer buf, queue q) { typedef std::vector (*partition_fn)(device dev); +static std::vector partition_affinity(device dev) { + std::vector subdevices = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::next_partitionable); + + return subdevices; +} + +// This is currently not tested static std::vector partition_equally(device dev) { std::vector subdevices = dev.create_sub_devices(1); @@ -46,7 +55,7 @@ static bool check_separate(device dev, buffer buf, assert(subsubdevices.size() > 1); // CHECK-SEPARATE: Create sub sub devices of sub device 0 // CHECK-SEPARATE: ---> piDevicePartition - + log_pi("Test sub sub device 0"); { queue q0(subsubdevices[0]); @@ -187,7 +196,7 @@ static bool check_fused_context(device dev, buffer buf, queue q1(fused_context, subsubdevices[1]); use_mem(buf, q1); } - // CHECK-FUSED: Test sub device 1 + // CHECK-FUSED: Test sub sub device 1 // CHECK-FUSED: ---> piQueueCreate // CHECK-FUSED: ---> piEnqueueKernelLaunch // CHECK-FUSED: ---> piEventsWait @@ -210,6 +219,8 @@ int main(int argc, const char **argv) { partition_fn partition; if (partition_type == "equally") { partition = partition_equally; + } else if (partition_type == "affinity") { + partition = partition_affinity; } else { assert(0 && "Unsupported partition type"); } From d53232df3342ebdf6e690a3ade93a6224c8599f6 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Thu, 15 Jul 2021 15:08:02 -0700 Subject: [PATCH 04/11] add new test. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 391 +++++++++++++-------------------- SYCL/Basic/subsubdevice_pi.cpp | 215 ++++++++++++++++++ 2 files changed, 372 insertions(+), 234 deletions(-) create mode 100644 SYCL/Basic/subsubdevice_pi.cpp diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index e9ef434aa2..513633e45a 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -1,243 +1,166 @@ // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out -// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE -// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate -// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// RUN: %HOST_RUN_PLACEHOLDER %t.out +// RUN: %CPU_RUN_PLACEHOLDER %t.out +// RUN: %GPU_RUN_PLACEHOLDER %t.out +// RUN: %ACC_RUN_PLACEHOLDER %t.out + +//==------------ subdevice.cpp - SYCL subdevice basic test -----------------==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// Intel OpenCL CPU Runtime supports device partition on all (multi-core) -// platforms. Other devices may not support this. +//===----------------------------------------------------------------------===// #include -#include -#include +#include +#include +#include +#include using namespace cl::sycl; -// Log to the same stream as SYCL_PI_TRACE -static void log_pi(const char *msg) { std::cout << msg << std::endl; } - -static void use_mem(buffer buf, queue q) { - q.submit([&](handler &cgh) { - auto acc = buf.get_access(cgh); - cgh.parallel_for(range<1>(buf.size()), - [=](item<1> itemID) { acc[itemID] += 1; }); - }); - q.wait(); -} - -typedef std::vector (*partition_fn)(device dev); - -static std::vector partition_affinity(device dev) { - std::vector subdevices = dev.create_sub_devices< - info::partition_property::partition_by_affinity_domain>( - info::partition_affinity_domain::next_partitionable); - - return subdevices; -} - -// This is currently not tested -static std::vector partition_equally(device dev) { - std::vector subdevices = - dev.create_sub_devices(1); - - return subdevices; -} - -static bool check_separate(device dev, buffer buf, - partition_fn partition) { - log_pi("Create sub devices"); - std::vector subdevices = partition(dev); - assert(subdevices.size() > 1); - // CHECK-SEPARATE: Create sub devices - // CHECK-SEPARATE: ---> piDevicePartition - - log_pi("Create sub sub devices of sub device 0"); - std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() > 1); - // CHECK-SEPARATE: Create sub sub devices of sub device 0 - // CHECK-SEPARATE: ---> piDevicePartition - - log_pi("Test sub sub device 0"); - { - queue q0(subsubdevices[0]); - use_mem(buf, q0); - } - // CHECK-SEPARATE: Test sub sub device 0 - // CHECK-SEPARATE: ---> piContextCreate - // CHECK-SEPARATE: ---> piQueueCreate - // CHECK-SEPARATE: ---> piMemBufferCreate - // CHECK-SEPARATE: ---> piEnqueueKernelLaunch - // CHECK-SEPARATE: ---> piEventsWait - - log_pi("Test sub sub device 1"); - { - queue q1(subsubdevices[1]); - use_mem(buf, q1); - } - // CHECK-SEPARATE: Test sub sub device 1 - // CHECK-SEPARATE: ---> piContextCreate - // CHECK-SEPARATE: ---> piQueueCreate - // CHECK-SEPARATE: ---> piMemBufferCreate - // - // Verify that we have a memcpy between subdevices in this case - // CHECK-SEPARATE: ---> piEnqueueMemBuffer{{Map|Read}} - // CHECK-SEPARATE: ---> piEnqueueMemBufferWrite - // - // CHECK-SEPARATE: ---> piEnqueueKernelLaunch - // CHECK-SEPARATE: ---> piEventsWait - - return true; -} - -static bool check_shared_context(device dev, buffer buf, - partition_fn partition) { - log_pi("Create sub devices"); - std::vector subdevices = partition(dev); - assert(subdevices.size() > 1); - // CHECK-SHARED: Create sub devices - // CHECK-SHARED: ---> piDevicePartition - - log_pi("Create sub sub devices of sub device 0"); - std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() > 1); - // CHECK-SHARED: Create sub sub devices of sub device 0 - // CHECK-SHARED: ---> piDevicePartition - - // Shared context: queues are bound to specific subdevices, but - // memory does not migrate - log_pi("Create shared context"); - context shared_context(subsubdevices); - // CHECK-SHARED: Create shared context - // CHECK-SHARED: ---> piContextCreate - // - // Make sure that a single context is created: see --implicit-check-not above. - - log_pi("Test sub sub device 0"); - { - queue q0(shared_context, subsubdevices[0]); - use_mem(buf, q0); - } - // CHECK-SHARED: Test sub sub device 0 - // CHECK-SHARED: ---> piQueueCreate - // CHECK-SHARED: ---> piMemBufferCreate - // - // Make sure that a single buffer is created (and shared between subdevices): - // see --implicit-check-not above. - // - // CHECK-SHARED: ---> piEnqueueKernelLaunch - // CHECK-SHARED: ---> piEventsWait - - log_pi("Test sub sub device 1"); - { - queue q1(shared_context, subsubdevices[1]); - use_mem(buf, q1); - } - // CHECK-SHARED: Test sub sub device 1 - // CHECK-SHARED: ---> piQueueCreate - // CHECK-SHARED: ---> piEnqueueKernelLaunch - // CHECK-SHARED: ---> piEventsWait - // CHECK-SHARED: ---> piEnqueueMemBufferRead - - return true; -} - -static bool check_fused_context(device dev, buffer buf, - partition_fn partition) { - log_pi("Create sub devices"); - std::vector subdevices = partition(dev); - assert(subdevices.size() > 1); - // CHECK-FUSED: Create sub devices - // CHECK-FUSED: ---> piDevicePartition - - log_pi("Create sub sub devices of sub device 0"); - std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() > 1); - // CHECK-FUSED: Create sub sub devices of sub device 0 - // CHECK-FUSED: ---> piDevicePartition - - // Fused context: same as shared context, but also includes the root device - log_pi("Create fused context"); - std::vector devices; - devices.push_back(dev); - devices.push_back(subdevices[0]); - devices.push_back(subdevices[1]); - context fused_context(devices); - // CHECK-FUSED: Create fused context - // CHECK-FUSED: ---> piContextCreate - // - // Make sure that a single context is created: see --implicit-check-not above. - - log_pi("Test root device"); - { - queue q(fused_context, dev); - use_mem(buf, q); - } - // CHECK-FUSED: Test root device - // CHECK-FUSED: ---> piQueueCreate - // CHECK-FUSED: ---> piMemBufferCreate - // - // Make sure that a single buffer is created (and shared between subdevices - // *and* the root device): see --implicit-check-not above. - // - // CHECK-FUSED: ---> piEnqueueKernelLaunch - // CHECK-FUSED: ---> piEventsWait - - log_pi("Test sub sub device 0"); - { - queue q0(fused_context, subsubdevices[0]); - use_mem(buf, q0); - } - // CHECK-FUSED: Test sub sub device 0 - // CHECK-FUSED: ---> piQueueCreate - // CHECK-FUSED: ---> piEnqueueKernelLaunch - // CHECK-FUSED: ---> piEventsWait - - log_pi("Test sub sub device 1"); - { - queue q1(fused_context, subsubdevices[1]); - use_mem(buf, q1); - } - // CHECK-FUSED: Test sub sub device 1 - // CHECK-FUSED: ---> piQueueCreate - // CHECK-FUSED: ---> piEnqueueKernelLaunch - // CHECK-FUSED: ---> piEventsWait - // CHECK-FUSED: ---> piEnqueueMemBufferRead - - return true; -} - -int main(int argc, const char **argv) { - assert(argc == 3 && "Invalid number of arguments"); - std::string test(argv[1]); - std::string partition_type(argv[2]); - - default_selector selector; - device dev(selector); - - std::vector host_mem(1024, 1); - buffer buf(&host_mem[0], host_mem.size()); - - partition_fn partition; - if (partition_type == "equally") { - partition = partition_equally; - } else if (partition_type == "affinity") { - partition = partition_affinity; - } else { - assert(0 && "Unsupported partition type"); - } - - bool result = false; - if (test == "separate") { - result = check_separate(dev, buf, partition); - } else if (test == "shared") { - result = check_shared_context(dev, buf, partition); - } else if (test == "fused") { - result = check_fused_context(dev, buf, partition); - } else { - assert(0 && "Unknown test"); - } - - if (!result) { - fprintf(stderr, "FAILED\n"); - return EXIT_FAILURE; +int main() { + try { + auto devices = device::get_devices(); + for (const auto &dev : devices) { + // TODO: implement subdevices creation for host device + if (dev.is_host()) + continue; + + assert(dev.get_info() == + info::partition_property::no_partition); + + size_t MaxSubDevices = + dev.get_info(); + + if (MaxSubDevices == 0) + continue; + + try { + auto SubDevicesEq = + dev.create_sub_devices( + 1); + assert(SubDevicesEq.size() == MaxSubDevices && + "Requested 1 compute unit in each subdevice, expected maximum " + "number of subdevices in output"); + std::cout << "Created " << SubDevicesEq.size() + << " subdevices using equal partition scheme" << std::endl; + + assert( + SubDevicesEq[0].get_info() == + info::partition_property::partition_equally); + + assert(SubDevicesEq[0].get_info().get() == + dev.get()); + } catch (feature_not_supported) { + // okay skip it + } + + try { + vector_class Counts(MaxSubDevices, 1); + auto SubDevicesByCount = dev.create_sub_devices< + info::partition_property::partition_by_counts>(Counts); + assert(SubDevicesByCount.size() == MaxSubDevices && + "Maximum number of subdevices was requested with 1 compute unit " + "on each"); + std::cout << "Created " << SubDevicesByCount.size() + << " subdevices using partition by counts scheme." + << std::endl; + assert(SubDevicesByCount[0] + .get_info() == + info::partition_property::partition_by_counts); + } catch (feature_not_supported) { + // okay skip it + } + + try { + auto SubDevicesDomainNuma = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::numa); + std::cout + << "Created " << SubDevicesDomainNuma.size() + << " subdevices using partition by numa affinity domain scheme." + << std::endl; + + auto SubSubDevicesDomainNuma = SubDevicesDomainNuma[0].create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::numa); + std::cout + << "Created " << SubSubDevicesDomainNuma.size() + << " sub-subdevices using partition by numa affinity domain scheme." + << std::endl; + } catch (feature_not_supported) { + // okay skip it + } + + try { + auto SubDevicesDomainL4 = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::L4_cache); + std::cout << "Created " << SubDevicesDomainL4.size() + << " subdevices using partition by L4 cache domain scheme." + << std::endl; + } catch (feature_not_supported) { + // okay skip it + } + + try { + auto SubDevicesDomainL3 = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::L3_cache); + std::cout << "Created " << SubDevicesDomainL3.size() + << " subdevices using partition by L3 cache domain scheme." + << std::endl; + } catch (feature_not_supported) { + // okay skip it + } + + try { + auto SubDevicesDomainL2 = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::L2_cache); + std::cout << "Created " << SubDevicesDomainL2.size() + << " subdevices using partition by L2 cache domain scheme." + << std::endl; + } catch (feature_not_supported) { + // okay skip it + } + + try { + auto SubDevicesDomainL1 = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::L1_cache); + std::cout << "Created " << SubDevicesDomainL1.size() + << " subdevices using partition by L1 cache domain scheme." + << std::endl; + } catch (feature_not_supported) { + // okay skip it + } + + try { + auto SubDevicesDomainNextPart = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::next_partitionable); + std::cout << "Created " << SubDevicesDomainNextPart.size() + << " subdevices using partition by next partitionable " + "domain scheme." + << std::endl; + + auto SubSubDevicesDomainNextPart = SubDevicesDomainNextPart[0].create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::next_partitionable); + std::cout << "Created " << SubSubDevicesDomainNextPart.size() + << " sub-subdevices from subdevice 0 using partition by next partitionable " + "domain scheme." + << std::endl; + } catch (feature_not_supported) { + // okay skip it + } + } + } catch (exception e) { + std::cout << "SYCL exception caught: " << e.what() << std::endl; + return 1; } + return 0; } diff --git a/SYCL/Basic/subsubdevice_pi.cpp b/SYCL/Basic/subsubdevice_pi.cpp new file mode 100644 index 0000000000..2966e57f0f --- /dev/null +++ b/SYCL/Basic/subsubdevice_pi.cpp @@ -0,0 +1,215 @@ +// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out +// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE +// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// +// Intel OpenCL CPU Runtime supports device partition on all (multi-core) +// platforms. Other devices may not support this. + +#include +#include +#include + +using namespace cl::sycl; + +// Log to the same stream as SYCL_PI_TRACE +static void log_pi(const char *msg) { std::cout << msg << std::endl; } + +static void use_mem(buffer buf, queue q) { + q.submit([&](handler &cgh) { + auto acc = buf.get_access(cgh); + cgh.parallel_for(range<1>(buf.size()), + [=](item<1> itemID) { acc[itemID] += 1; }); + }); + q.wait(); +} + +typedef std::vector (*partition_fn)(device dev); + +static std::vector partition_affinity(device dev) { + std::vector subdevices = dev.create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::next_partitionable); + + return subdevices; +} + +// This is currently not tested +static std::vector partition_equally(device dev) { + std::vector subdevices = + dev.create_sub_devices(1); + + return subdevices; +} + +static bool check_separate(device dev, buffer buf, + partition_fn partition) { + log_pi("Create sub devices"); + std::vector subdevices = partition(dev); + assert(subdevices.size() > 1); + // CHECK-SEPARATE: Create sub devices + // CHECK-SEPARATE: ---> piDevicePartition + + log_pi("Create sub sub devices of sub device 0"); + std::vector subsubdevices = partition(subdevices[0]); + assert(subsubdevices.size() >= 1); + // CHECK-SEPARATE: Create sub sub devices of sub device 0 + // CHECK-SEPARATE: ---> piDevicePartition + + log_pi("Test sub sub device 0"); + { + queue q0(subsubdevices[0]); + use_mem(buf, q0); + } + // CHECK-SEPARATE: Test sub sub device 0 + // CHECK-SEPARATE: ---> piContextCreate + // CHECK-SEPARATE: ---> piQueueCreate + // CHECK-SEPARATE: ---> piMemBufferCreate + // CHECK-SEPARATE: ---> piEnqueueKernelLaunch + // CHECK-SEPARATE: ---> piEventsWait + + return true; +} + +static bool check_shared_context(device dev, buffer buf, + partition_fn partition) { + log_pi("Create sub devices"); + std::vector subdevices = partition(dev); + assert(subdevices.size() > 1); + // CHECK-SHARED: Create sub devices + // CHECK-SHARED: ---> piDevicePartition + + log_pi("Create sub sub devices of sub device 0"); + std::vector subsubdevices = partition(subdevices[0]); + assert(subsubdevices.size() >= 1); + // CHECK-SHARED: Create sub sub devices of sub device 0 + // CHECK-SHARED: ---> piDevicePartition + + // Shared context: queues are bound to specific subdevices, but + // memory does not migrate + log_pi("Create shared context"); + context shared_context(subsubdevices); + // CHECK-SHARED: Create shared context + // CHECK-SHARED: ---> piContextCreate + // + // Make sure that a single context is created: see --implicit-check-not above. + + log_pi("Test sub sub device 0"); + { + queue q0(shared_context, subsubdevices[0]); + use_mem(buf, q0); + } + // CHECK-SHARED: Test sub sub device 0 + // CHECK-SHARED: ---> piQueueCreate + // CHECK-SHARED: ---> piMemBufferCreate + // + // Make sure that a single buffer is created (and shared between subdevices): + // see --implicit-check-not above. + // + // CHECK-SHARED: ---> piEnqueueKernelLaunch + // CHECK-SHARED: ---> piEventsWait + + return true; +} + +static bool check_fused_context(device dev, buffer buf, + partition_fn partition) { + log_pi("Create sub devices"); + std::vector subdevices = partition(dev); + assert(subdevices.size() > 1); + // CHECK-FUSED: Create sub devices + // CHECK-FUSED: ---> piDevicePartition + + log_pi("Create sub sub devices of sub device 0"); + std::vector subsubdevices = partition(subdevices[0]); + assert(subsubdevices.size() >= 1); + // CHECK-FUSED: Create sub sub devices of sub device 0 + // CHECK-FUSED: ---> piDevicePartition + + // Fused context: same as shared context, but also includes the root device + log_pi("Create fused context"); + std::vector devices; + devices.push_back(dev); + devices.push_back(subdevices[0]); + devices.push_back(subsubdevices[0]); + context fused_context(devices); + // CHECK-FUSED: Create fused context + // CHECK-FUSED: ---> piContextCreate + // + // Make sure that a single context is created: see --implicit-check-not above. + + log_pi("Test root device"); + { + queue q(fused_context, dev); + use_mem(buf, q); + } + // CHECK-FUSED: Test root device + // CHECK-FUSED: ---> piQueueCreate + // CHECK-FUSED: ---> piMemBufferCreate + // + // Make sure that a single buffer is created (and shared between subdevices + // *and* the root device): see --implicit-check-not above. + // + // CHECK-FUSED: ---> piEnqueueKernelLaunch + // CHECK-FUSED: ---> piQueueFinish + + log_pi("Test sub device 0"); + { + queue q0(fused_context, subdevices[0]); + use_mem(buf, q0); + } + // CHECK-FUSED: Test sub device 0 + // CHECK-FUSED: ---> piQueueCreate + // CHECK-FUSED: ---> piEnqueueKernelLaunch + // CHECK-FUSED: ---> piQueueFinish + + log_pi("Test sub sub device 0"); + { + queue q1(fused_context, subsubdevices[0]); + use_mem(buf, q1); + } + // CHECK-FUSED: Test sub sub device 0 + // CHECK-FUSED: ---> piQueueCreate + // CHECK-FUSED: ---> piEnqueueKernelLaunch + // CHECK-FUSED: ---> piQueueFinish + // CHECK-FUSED: ---> piEnqueueMemBufferRead + + return true; +} + +int main(int argc, const char **argv) { + assert(argc == 3 && "Invalid number of arguments"); + std::string test(argv[1]); + std::string partition_type(argv[2]); + + default_selector selector; + device dev(selector); + + std::vector host_mem(1024, 1); + buffer buf(&host_mem[0], host_mem.size()); + + partition_fn partition; + if (partition_type == "equally") { + partition = partition_equally; + } else if (partition_type == "affinity") { + partition = partition_affinity; + } else { + assert(0 && "Unsupported partition type"); + } + + bool result = false; + if (test == "separate") { + result = check_separate(dev, buf, partition); + } else if (test == "shared") { + result = check_shared_context(dev, buf, partition); + } else if (test == "fused") { + result = check_fused_context(dev, buf, partition); + } else { + assert(0 && "Unknown test"); + } + + if (!result) { + fprintf(stderr, "FAILED\n"); + return EXIT_FAILURE; + } +} From 0b1aa5a9b6f96cbae3c88a7a8b9de19de6f50422 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Thu, 15 Jul 2021 15:43:13 -0700 Subject: [PATCH 05/11] clang formatted. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index 513633e45a..912b0b57c7 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -83,9 +83,11 @@ int main() { << " subdevices using partition by numa affinity domain scheme." << std::endl; - auto SubSubDevicesDomainNuma = SubDevicesDomainNuma[0].create_sub_devices< - info::partition_property::partition_by_affinity_domain>( - info::partition_affinity_domain::numa); + auto SubSubDevicesDomainNuma = + SubDevicesDomainNuma[0] + .create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::numa); std::cout << "Created " << SubSubDevicesDomainNuma.size() << " sub-subdevices using partition by numa affinity domain scheme." @@ -147,12 +149,14 @@ int main() { "domain scheme." << std::endl; - auto SubSubDevicesDomainNextPart = SubDevicesDomainNextPart[0].create_sub_devices< - info::partition_property::partition_by_affinity_domain>( - info::partition_affinity_domain::next_partitionable); + auto SubSubDevicesDomainNextPart = + SubDevicesDomainNextPart[0] + .create_sub_devices< + info::partition_property::partition_by_affinity_domain>( + info::partition_affinity_domain::next_partitionable); std::cout << "Created " << SubSubDevicesDomainNextPart.size() - << " sub-subdevices from subdevice 0 using partition by next partitionable " - "domain scheme." + << " sub-subdevices from subdevice 0 using partition by next " + "partitionable domain scheme." << std::endl; } catch (feature_not_supported) { // okay skip it From 6043755e9fb6181b3dcb0185b87dc7a3a5534040 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Thu, 15 Jul 2021 16:01:30 -0700 Subject: [PATCH 06/11] clang formatted. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index 912b0b57c7..fcf769171f 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -83,7 +83,7 @@ int main() { << " subdevices using partition by numa affinity domain scheme." << std::endl; - auto SubSubDevicesDomainNuma = + auto SubSubDevicesDomainNuma = SubDevicesDomainNuma[0] .create_sub_devices< info::partition_property::partition_by_affinity_domain>( From f5b8c83a369701c6f5a4fbd91658ce5cf7b8b390 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Fri, 16 Jul 2021 21:44:51 -0700 Subject: [PATCH 07/11] modify context Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 3 ++- SYCL/Basic/subsubdevice_pi.cpp | 26 +++++++++++++++++--------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index fcf769171f..8004cb7bb2 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -90,7 +90,8 @@ int main() { info::partition_affinity_domain::numa); std::cout << "Created " << SubSubDevicesDomainNuma.size() - << " sub-subdevices using partition by numa affinity domain scheme." + << " sub-subdevices from subdevice 0 using partition by numa " + "affinity domain scheme." << std::endl; } catch (feature_not_supported) { // okay skip it diff --git a/SYCL/Basic/subsubdevice_pi.cpp b/SYCL/Basic/subsubdevice_pi.cpp index 2966e57f0f..fd37268ed5 100644 --- a/SYCL/Basic/subsubdevice_pi.cpp +++ b/SYCL/Basic/subsubdevice_pi.cpp @@ -2,9 +2,6 @@ // RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE // RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate // RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate -// -// Intel OpenCL CPU Runtime supports device partition on all (multi-core) -// platforms. Other devices may not support this. #include #include @@ -52,7 +49,7 @@ static bool check_separate(device dev, buffer buf, log_pi("Create sub sub devices of sub device 0"); std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() >= 1); + assert(subsubdevices.size() > 1); // CHECK-SEPARATE: Create sub sub devices of sub device 0 // CHECK-SEPARATE: ---> piDevicePartition @@ -81,7 +78,7 @@ static bool check_shared_context(device dev, buffer buf, log_pi("Create sub sub devices of sub device 0"); std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() >= 1); + assert(subsubdevices.size() > 1); // CHECK-SHARED: Create sub sub devices of sub device 0 // CHECK-SHARED: ---> piDevicePartition @@ -96,8 +93,8 @@ static bool check_shared_context(device dev, buffer buf, log_pi("Test sub sub device 0"); { - queue q0(shared_context, subsubdevices[0]); - use_mem(buf, q0); + queue q(shared_context, subsubdevices[0]); + use_mem(buf, q); } // CHECK-SHARED: Test sub sub device 0 // CHECK-SHARED: ---> piQueueCreate @@ -107,7 +104,18 @@ static bool check_shared_context(device dev, buffer buf, // see --implicit-check-not above. // // CHECK-SHARED: ---> piEnqueueKernelLaunch - // CHECK-SHARED: ---> piEventsWait + // CHECK-SHARED: ---> piQueueFinish + + log_pi("Test sub sub device 1"); + { + queue q1(shared_context, subsubdevices[1]); + use_mem(buf, q1); + } + // CHECK-SHARED: Test sub sub device 1 + // CHECK-SHARED: ---> piQueueCreate + // CHECK-SHARED: ---> piEnqueueKernelLaunch + // CHECK-SHARED: ---> piQueueFinish + // CHECK-SHARED: ---> piEnqueueMemBufferRead return true; } @@ -122,7 +130,7 @@ static bool check_fused_context(device dev, buffer buf, log_pi("Create sub sub devices of sub device 0"); std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() >= 1); + assert(subsubdevices.size() > 1); // CHECK-FUSED: Create sub sub devices of sub device 0 // CHECK-FUSED: ---> piDevicePartition From 9ce174c6660926f851d8bbbc7710c9f31a272037 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Fri, 16 Jul 2021 22:26:50 -0700 Subject: [PATCH 08/11] clang formatted. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/SYCL/Basic/subsubdevice.cpp b/SYCL/Basic/subsubdevice.cpp index 8004cb7bb2..b9f400b6a3 100644 --- a/SYCL/Basic/subsubdevice.cpp +++ b/SYCL/Basic/subsubdevice.cpp @@ -88,11 +88,11 @@ int main() { .create_sub_devices< info::partition_property::partition_by_affinity_domain>( info::partition_affinity_domain::numa); - std::cout - << "Created " << SubSubDevicesDomainNuma.size() - << " sub-subdevices from subdevice 0 using partition by numa " - "affinity domain scheme." - << std::endl; + + std::cout << "Created " << SubSubDevicesDomainNuma.size() + << " sub-subdevices from subdevice 0 using partition by numa " + "affinity domain scheme." + << std::endl; } catch (feature_not_supported) { // okay skip it } From 3a595a6df9ae07213fee7e1b96ff62b6c080e809 Mon Sep 17 00:00:00 2001 From: rehana begam Date: Wed, 21 Jul 2021 10:13:50 -0700 Subject: [PATCH 09/11] update comment. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice_pi.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SYCL/Basic/subsubdevice_pi.cpp b/SYCL/Basic/subsubdevice_pi.cpp index fd37268ed5..6b95e55fb6 100644 --- a/SYCL/Basic/subsubdevice_pi.cpp +++ b/SYCL/Basic/subsubdevice_pi.cpp @@ -31,7 +31,7 @@ static std::vector partition_affinity(device dev) { return subdevices; } -// This is currently not tested +// Equality is currently not tested static std::vector partition_equally(device dev) { std::vector subdevices = dev.create_sub_devices(1); From f18cb47df08afe551b33dd6c8d0e6eee4b2f0dad Mon Sep 17 00:00:00 2001 From: rehana begam Date: Wed, 21 Jul 2021 11:16:38 -0700 Subject: [PATCH 10/11] adds requirement. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice_pi.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/SYCL/Basic/subsubdevice_pi.cpp b/SYCL/Basic/subsubdevice_pi.cpp index 6b95e55fb6..14f50d4cba 100644 --- a/SYCL/Basic/subsubdevice_pi.cpp +++ b/SYCL/Basic/subsubdevice_pi.cpp @@ -2,6 +2,7 @@ // RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE // RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate // RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate +// REQUIRES: level_zero #include #include From 7ce153631213d6cf770d1edd9a6044f880b7441f Mon Sep 17 00:00:00 2001 From: rehana begam Date: Wed, 21 Jul 2021 15:41:16 -0700 Subject: [PATCH 11/11] removing the 2nd test. Signed-off-by: rehana begam --- SYCL/Basic/subsubdevice_pi.cpp | 224 --------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 SYCL/Basic/subsubdevice_pi.cpp diff --git a/SYCL/Basic/subsubdevice_pi.cpp b/SYCL/Basic/subsubdevice_pi.cpp deleted file mode 100644 index 14f50d4cba..0000000000 --- a/SYCL/Basic/subsubdevice_pi.cpp +++ /dev/null @@ -1,224 +0,0 @@ -// RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out -// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE -// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate -// RUN: %GPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused affinity %GPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate -// REQUIRES: level_zero - -#include -#include -#include - -using namespace cl::sycl; - -// Log to the same stream as SYCL_PI_TRACE -static void log_pi(const char *msg) { std::cout << msg << std::endl; } - -static void use_mem(buffer buf, queue q) { - q.submit([&](handler &cgh) { - auto acc = buf.get_access(cgh); - cgh.parallel_for(range<1>(buf.size()), - [=](item<1> itemID) { acc[itemID] += 1; }); - }); - q.wait(); -} - -typedef std::vector (*partition_fn)(device dev); - -static std::vector partition_affinity(device dev) { - std::vector subdevices = dev.create_sub_devices< - info::partition_property::partition_by_affinity_domain>( - info::partition_affinity_domain::next_partitionable); - - return subdevices; -} - -// Equality is currently not tested -static std::vector partition_equally(device dev) { - std::vector subdevices = - dev.create_sub_devices(1); - - return subdevices; -} - -static bool check_separate(device dev, buffer buf, - partition_fn partition) { - log_pi("Create sub devices"); - std::vector subdevices = partition(dev); - assert(subdevices.size() > 1); - // CHECK-SEPARATE: Create sub devices - // CHECK-SEPARATE: ---> piDevicePartition - - log_pi("Create sub sub devices of sub device 0"); - std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() > 1); - // CHECK-SEPARATE: Create sub sub devices of sub device 0 - // CHECK-SEPARATE: ---> piDevicePartition - - log_pi("Test sub sub device 0"); - { - queue q0(subsubdevices[0]); - use_mem(buf, q0); - } - // CHECK-SEPARATE: Test sub sub device 0 - // CHECK-SEPARATE: ---> piContextCreate - // CHECK-SEPARATE: ---> piQueueCreate - // CHECK-SEPARATE: ---> piMemBufferCreate - // CHECK-SEPARATE: ---> piEnqueueKernelLaunch - // CHECK-SEPARATE: ---> piEventsWait - - return true; -} - -static bool check_shared_context(device dev, buffer buf, - partition_fn partition) { - log_pi("Create sub devices"); - std::vector subdevices = partition(dev); - assert(subdevices.size() > 1); - // CHECK-SHARED: Create sub devices - // CHECK-SHARED: ---> piDevicePartition - - log_pi("Create sub sub devices of sub device 0"); - std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() > 1); - // CHECK-SHARED: Create sub sub devices of sub device 0 - // CHECK-SHARED: ---> piDevicePartition - - // Shared context: queues are bound to specific subdevices, but - // memory does not migrate - log_pi("Create shared context"); - context shared_context(subsubdevices); - // CHECK-SHARED: Create shared context - // CHECK-SHARED: ---> piContextCreate - // - // Make sure that a single context is created: see --implicit-check-not above. - - log_pi("Test sub sub device 0"); - { - queue q(shared_context, subsubdevices[0]); - use_mem(buf, q); - } - // CHECK-SHARED: Test sub sub device 0 - // CHECK-SHARED: ---> piQueueCreate - // CHECK-SHARED: ---> piMemBufferCreate - // - // Make sure that a single buffer is created (and shared between subdevices): - // see --implicit-check-not above. - // - // CHECK-SHARED: ---> piEnqueueKernelLaunch - // CHECK-SHARED: ---> piQueueFinish - - log_pi("Test sub sub device 1"); - { - queue q1(shared_context, subsubdevices[1]); - use_mem(buf, q1); - } - // CHECK-SHARED: Test sub sub device 1 - // CHECK-SHARED: ---> piQueueCreate - // CHECK-SHARED: ---> piEnqueueKernelLaunch - // CHECK-SHARED: ---> piQueueFinish - // CHECK-SHARED: ---> piEnqueueMemBufferRead - - return true; -} - -static bool check_fused_context(device dev, buffer buf, - partition_fn partition) { - log_pi("Create sub devices"); - std::vector subdevices = partition(dev); - assert(subdevices.size() > 1); - // CHECK-FUSED: Create sub devices - // CHECK-FUSED: ---> piDevicePartition - - log_pi("Create sub sub devices of sub device 0"); - std::vector subsubdevices = partition(subdevices[0]); - assert(subsubdevices.size() > 1); - // CHECK-FUSED: Create sub sub devices of sub device 0 - // CHECK-FUSED: ---> piDevicePartition - - // Fused context: same as shared context, but also includes the root device - log_pi("Create fused context"); - std::vector devices; - devices.push_back(dev); - devices.push_back(subdevices[0]); - devices.push_back(subsubdevices[0]); - context fused_context(devices); - // CHECK-FUSED: Create fused context - // CHECK-FUSED: ---> piContextCreate - // - // Make sure that a single context is created: see --implicit-check-not above. - - log_pi("Test root device"); - { - queue q(fused_context, dev); - use_mem(buf, q); - } - // CHECK-FUSED: Test root device - // CHECK-FUSED: ---> piQueueCreate - // CHECK-FUSED: ---> piMemBufferCreate - // - // Make sure that a single buffer is created (and shared between subdevices - // *and* the root device): see --implicit-check-not above. - // - // CHECK-FUSED: ---> piEnqueueKernelLaunch - // CHECK-FUSED: ---> piQueueFinish - - log_pi("Test sub device 0"); - { - queue q0(fused_context, subdevices[0]); - use_mem(buf, q0); - } - // CHECK-FUSED: Test sub device 0 - // CHECK-FUSED: ---> piQueueCreate - // CHECK-FUSED: ---> piEnqueueKernelLaunch - // CHECK-FUSED: ---> piQueueFinish - - log_pi("Test sub sub device 0"); - { - queue q1(fused_context, subsubdevices[0]); - use_mem(buf, q1); - } - // CHECK-FUSED: Test sub sub device 0 - // CHECK-FUSED: ---> piQueueCreate - // CHECK-FUSED: ---> piEnqueueKernelLaunch - // CHECK-FUSED: ---> piQueueFinish - // CHECK-FUSED: ---> piEnqueueMemBufferRead - - return true; -} - -int main(int argc, const char **argv) { - assert(argc == 3 && "Invalid number of arguments"); - std::string test(argv[1]); - std::string partition_type(argv[2]); - - default_selector selector; - device dev(selector); - - std::vector host_mem(1024, 1); - buffer buf(&host_mem[0], host_mem.size()); - - partition_fn partition; - if (partition_type == "equally") { - partition = partition_equally; - } else if (partition_type == "affinity") { - partition = partition_affinity; - } else { - assert(0 && "Unsupported partition type"); - } - - bool result = false; - if (test == "separate") { - result = check_separate(dev, buf, partition); - } else if (test == "shared") { - result = check_shared_context(dev, buf, partition); - } else if (test == "fused") { - result = check_fused_context(dev, buf, partition); - } else { - assert(0 && "Unknown test"); - } - - if (!result) { - fprintf(stderr, "FAILED\n"); - return EXIT_FAILURE; - } -}