From 4dc4233293f82bdf980a08ce0df1f50f6bdeb5b7 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Wed, 18 Jun 2025 16:57:57 +0100 Subject: [PATCH 1/7] Add reference counting to handle_base struct to allow all UR handles to inherit common reference counting functionality. Update L0 and L0V2 to use this. --- .../source/adapters/level_zero/adapter.cpp | 12 ++--- .../source/adapters/level_zero/adapter.hpp | 2 - .../adapters/level_zero/async_alloc.cpp | 2 +- .../adapters/level_zero/command_buffer.cpp | 10 ++-- .../source/adapters/level_zero/common.hpp | 48 +------------------ .../source/adapters/level_zero/context.cpp | 6 +-- .../source/adapters/level_zero/device.cpp | 6 +-- .../source/adapters/level_zero/event.cpp | 16 +++---- .../source/adapters/level_zero/kernel.cpp | 6 +-- .../source/adapters/level_zero/memory.cpp | 10 ++-- .../source/adapters/level_zero/memory.hpp | 2 +- .../adapters/level_zero/physical_mem.cpp | 6 +-- .../source/adapters/level_zero/program.cpp | 8 ++-- .../source/adapters/level_zero/queue.cpp | 18 +++---- .../source/adapters/level_zero/sampler.cpp | 4 +- .../source/adapters/level_zero/usm.cpp | 10 ++-- .../adapters/level_zero/v2/command_buffer.cpp | 6 +-- .../level_zero/v2/command_list_manager.cpp | 2 +- .../source/adapters/level_zero/v2/context.cpp | 6 +-- .../source/adapters/level_zero/v2/event.cpp | 6 +-- .../adapters/level_zero/v2/event_pool.cpp | 4 +- .../source/adapters/level_zero/v2/kernel.cpp | 6 +-- .../source/adapters/level_zero/v2/memory.cpp | 6 +-- .../adapters/level_zero/v2/queue_handle.hpp | 4 +- .../v2/queue_immediate_in_order.cpp | 2 +- .../source/adapters/level_zero/v2/usm.cpp | 6 +-- unified-runtime/source/ur/ur.hpp | 8 ++++ 27 files changed, 91 insertions(+), 131 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 6b23d0161a4f5..25edec5417df9 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -675,7 +675,7 @@ ur_result_t urAdapterGet( } *Adapters = GlobalAdapter; - if (GlobalAdapter->RefCount++ == 0) { + if (GlobalAdapter->incrementRefCount() == 0) { adapterStateInit(); } } @@ -692,7 +692,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { // NOTE: This does not require guarding with a mutex; the instant the ref // count hits zero, both Get and Retain are UB. - if (--GlobalAdapter->RefCount == 0) { + if (GlobalAdapter->decrementRefCount() == 0) { auto result = adapterStateTeardown(); #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's @@ -709,9 +709,9 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { return UR_RESULT_SUCCESS; } -ur_result_t urAdapterRetain([[maybe_unused]] ur_adapter_handle_t Adapter) { +ur_result_t urAdapterRetain(ur_adapter_handle_t) { assert(GlobalAdapter && GlobalAdapter == Adapter); - GlobalAdapter->RefCount++; + GlobalAdapter->incrementRefCount(); return UR_RESULT_SUCCESS; } @@ -740,12 +740,12 @@ ur_result_t urAdapterGetInfo(ur_adapter_handle_t, ur_adapter_info_t PropName, case UR_ADAPTER_INFO_BACKEND: return ReturnValue(UR_BACKEND_LEVEL_ZERO); case UR_ADAPTER_INFO_REFERENCE_COUNT: - return ReturnValue(GlobalAdapter->RefCount.load()); + return ReturnValue(GlobalAdapter->getRefCount()); case UR_ADAPTER_INFO_VERSION: { #ifdef UR_ADAPTER_LEVEL_ZERO_V2 uint32_t adapterVersion = 2; #else - uint32_t adapterVersion = 1; + uint32_t adapterVersion = 1; #endif return ReturnValue(adapterVersion); } diff --git a/unified-runtime/source/adapters/level_zero/adapter.hpp b/unified-runtime/source/adapters/level_zero/adapter.hpp index bb0a9058bce1b..59fd0971cd20f 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.hpp +++ b/unified-runtime/source/adapters/level_zero/adapter.hpp @@ -11,7 +11,6 @@ #include "logger/ur_logger.hpp" #include "ur_interface_loader.hpp" -#include #include #include #include @@ -26,7 +25,6 @@ class ur_legacy_sink; struct ur_adapter_handle_t_ : ur::handle_base { ur_adapter_handle_t_(); - std::atomic RefCount = 0; zes_pfnDriverGetDeviceByUuidExp_t getDeviceByUUIdFunctionPtr = nullptr; zes_pfnDriverGet_t getSysManDriversFunctionPtr = nullptr; diff --git a/unified-runtime/source/adapters/level_zero/async_alloc.cpp b/unified-runtime/source/adapters/level_zero/async_alloc.cpp index 204b43c3bcc79..96729c9da0f5d 100644 --- a/unified-runtime/source/adapters/level_zero/async_alloc.cpp +++ b/unified-runtime/source/adapters/level_zero/async_alloc.cpp @@ -247,7 +247,7 @@ ur_result_t urEnqueueUSMFreeExp( } size_t size = umfPoolMallocUsableSize(hPool, Mem); - (*Event)->RefCount.increment(); + (*Event)->incrementRefCount(); usmPool->AsyncPool.insert(Mem, size, *Event, Queue); // Signal that USM free event was finished diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index 0773eadad7fd8..108093764c1cd 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -840,13 +840,13 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) { - CommandBuffer->RefCount.increment(); + CommandBuffer->incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) { - if (!CommandBuffer->RefCount.decrementAndTest()) + if (!CommandBuffer->decrementRefCount() == 0) return UR_RESULT_SUCCESS; UR_CALL(waitForOngoingExecution(CommandBuffer)); @@ -1641,7 +1641,7 @@ ur_result_t enqueueImmediateAppendPath( if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->RefCount.increment(); + (*Event)->incrementRefCount(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(CommandListHelper, false, false)); @@ -1724,7 +1724,7 @@ ur_result_t enqueueWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer, if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->RefCount.increment(); + (*Event)->incrementRefCount(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(SignalCommandList, false /*IsBlocking*/, @@ -1848,7 +1848,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hCommandBuffer->RefCount.load()}); + return ReturnValue(uint32_t{hCommandBuffer->getRefCount()}); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index 19e22de14605d..7702332e3d8de 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -220,55 +220,9 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString); #define ZE_CALL_NOCHECK_NAME(ZeName, ZeArgs, callName) \ ZeCall().doCall(ZeName ZeArgs, callName, #ZeArgs, false) -// This wrapper around std::atomic is created to limit operations with reference -// counter and to make allowed operations more transparent in terms of -// thread-safety in the plugin. increment() and load() operations do not need a -// mutex guard around them since the underlying data is already atomic. -// decrementAndTest() method is used to guard a code which needs to be -// executed when object's ref count becomes zero after release. This method also -// doesn't need a mutex guard because decrement operation is atomic and only one -// thread can reach ref count equal to zero, i.e. only a single thread can pass -// through this check. -struct ReferenceCounter { - ReferenceCounter() : RefCount{1} {} - - // Reset the counter to the initial value. - void reset() { RefCount = 1; } - - // Used when retaining an object. - void increment() { RefCount++; } - - // Supposed to be used in ur*GetInfo* methods where ref count value is - // requested. - uint32_t load() { return RefCount.load(); } - - // This method allows to guard a code which needs to be executed when object's - // ref count becomes zero after release. It is important to notice that only a - // single thread can pass through this check. This is true because of several - // reasons: - // 1. Decrement operation is executed atomically. - // 2. It is not allowed to retain an object after its refcount reaches zero. - // 3. It is not allowed to release an object more times than the value of - // the ref count. - // 2. and 3. basically means that we can't use an object at all as soon as its - // refcount reaches zero. Using this check guarantees that code for deleting - // an object and releasing its resources is executed once by a single thread - // and we don't need to use any mutexes to guard access to this object in the - // scope after this check. Of course if we access another objects in this code - // (not the one which is being deleted) then access to these objects must be - // guarded, for example with a mutex. - bool decrementAndTest() { return --RefCount == 0; } - -private: - std::atomic RefCount; -}; - // Base class to store common data struct ur_object : ur::handle_base { - ur_object() : handle_base(), RefCount{} {} - - // Must be atomic to prevent data race when incrementing/decrementing. - ReferenceCounter RefCount; + ur_object() : handle_base() {} // This mutex protects accesses to all the non-const member variables. // Exclusive access is required to modify any of these members. diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index 3209b8b789155..6c9553001ce5d 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -61,7 +61,7 @@ ur_result_t urContextRetain( /// [in] handle of the context to get a reference of. ur_context_handle_t Context) { - Context->RefCount.increment(); + Context->incrementRefCount(); return UR_RESULT_SUCCESS; } @@ -113,7 +113,7 @@ ur_result_t urContextGetInfo( case UR_CONTEXT_INFO_NUM_DEVICES: return ReturnValue(uint32_t(Context->Devices.size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Context->RefCount.load()}); + return ReturnValue(uint32_t{Context->getRefCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: // 2D USM memcpy is supported. return ReturnValue(uint8_t{UseMemcpy2DOperations}); @@ -251,7 +251,7 @@ ur_device_handle_t ur_context_handle_t_::getRootDevice() const { // from the list of tracked contexts. ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { - if (!Context->RefCount.decrementAndTest()) + if (!Context->decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (IndirectAccessTrackingEnabled) { diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 6392b3802a199..0b85b5845da01 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -470,7 +470,7 @@ ur_result_t urDeviceGetInfo( return ReturnValue((uint32_t)Device->SubDevices.size()); } case UR_DEVICE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Device->RefCount.load()}); + return ReturnValue(uint32_t{Device->getRefCount()}); case UR_DEVICE_INFO_SUPPORTED_PARTITIONS: { // SYCL spec says: if this SYCL device cannot be partitioned into at least // two sub devices then the returned vector must be empty. @@ -1666,7 +1666,7 @@ ur_result_t urDeviceGetGlobalTimestamps( ur_result_t urDeviceRetain(ur_device_handle_t Device) { // The root-device ref-count remains unchanged (always 1). if (Device->isSubDevice()) { - Device->RefCount.increment(); + Device->incrementRefCount(); } return UR_RESULT_SUCCESS; } @@ -1674,7 +1674,7 @@ ur_result_t urDeviceRetain(ur_device_handle_t Device) { ur_result_t urDeviceRelease(ur_device_handle_t Device) { // Root devices are destroyed during the piTearDown process. if (Device->isSubDevice()) { - if (Device->RefCount.decrementAndTest()) { + if (Device->decrementRefCount() == 0) { delete Device; } } diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index f06cae5ec0cb3..2ff41ea6dc968 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -505,7 +505,7 @@ ur_result_t urEventGetInfo( return ReturnValue(Result); } case UR_EVENT_INFO_REFERENCE_COUNT: { - return ReturnValue(Event->RefCount.load()); + return ReturnValue(Event->getRefCount()); } default: UR_LOG(ERR, "Unsupported ParamName in urEventGetInfo: ParamName={}(0x{})", @@ -874,7 +874,7 @@ ur_result_t /// [in] handle of the event object urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { Event->RefCountExternal++; - Event->RefCount.increment(); + Event->incrementRefCount(); return UR_RESULT_SUCCESS; } @@ -1088,7 +1088,7 @@ ur_event_handle_t_::~ur_event_handle_t_() { ur_result_t urEventReleaseInternal(ur_event_handle_t Event, bool *isEventDeleted) { - if (!Event->RefCount.decrementAndTest()) + if (!Event->decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (Event->OriginAllocEvent) { @@ -1429,7 +1429,7 @@ ur_result_t ur_event_handle_t_::reset() { CommandType = UR_EXT_COMMAND_TYPE_USER; WaitList = {}; RefCountExternal = 0; - RefCount.reset(); + resetRefCount(); CommandList = std::nullopt; completionBatch = std::nullopt; OriginAllocEvent = nullptr; @@ -1524,7 +1524,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( std::shared_lock Lock(CurQueue->LastCommandEvent->Mutex); this->ZeEventList[0] = CurQueue->LastCommandEvent->ZeEvent; this->UrEventList[0] = CurQueue->LastCommandEvent; - this->UrEventList[0]->RefCount.increment(); + this->UrEventList[0]->incrementRefCount(); TmpListLength = 1; } else if (EventListLength > 0) { this->ZeEventList = new ze_event_handle_t[EventListLength]; @@ -1660,7 +1660,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( IsInternal, IsMultiDevice)); MultiDeviceZeEvent = MultiDeviceEvent->ZeEvent; const auto &ZeCommandList = CommandList->first; - EventList[I]->RefCount.increment(); + EventList[I]->incrementRefCount(); // Append a Barrier to wait on the original event while signalling the // new multi device event. @@ -1676,11 +1676,11 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( this->ZeEventList[TmpListLength] = MultiDeviceZeEvent; this->UrEventList[TmpListLength] = MultiDeviceEvent; - this->UrEventList[TmpListLength]->RefCount.increment(); + this->UrEventList[TmpListLength]->incrementRefCount(); } else { this->ZeEventList[TmpListLength] = EventList[I]->ZeEvent; this->UrEventList[TmpListLength] = EventList[I]; - this->UrEventList[TmpListLength]->RefCount.increment(); + this->UrEventList[TmpListLength]->incrementRefCount(); } if (QueueLock.has_value()) { diff --git a/unified-runtime/source/adapters/level_zero/kernel.cpp b/unified-runtime/source/adapters/level_zero/kernel.cpp index 29c6b19e2bbfe..64c4594759da9 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/kernel.cpp @@ -787,7 +787,7 @@ ur_result_t urKernelGetInfo( } } case UR_KERNEL_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Kernel->RefCount.load()}); + return ReturnValue(uint32_t{Kernel->getRefCount()}); case UR_KERNEL_INFO_ATTRIBUTES: try { uint32_t Size; @@ -938,7 +938,7 @@ ur_result_t urKernelGetSubGroupInfo( ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t Kernel) { - Kernel->RefCount.increment(); + Kernel->incrementRefCount(); return UR_RESULT_SUCCESS; } @@ -946,7 +946,7 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release ur_kernel_handle_t Kernel) { - if (!Kernel->RefCount.decrementAndTest()) + if (!Kernel->decrementRefCount() == 0) return UR_RESULT_SUCCESS; auto KernelProgram = Kernel->Program; diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index 0f6bb37dde904..af17d325173cd 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -1052,7 +1052,7 @@ ur_result_t urEnqueueMemBufferMap( // Add the event to the command list. CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->RefCount.increment(); + (*Event)->incrementRefCount(); const auto &ZeCommandList = CommandList->first; const auto &WaitList = (*Event)->WaitList; @@ -1183,7 +1183,7 @@ ur_result_t urEnqueueMemUnmap( nullptr /*ForcedCmdQueue*/)); CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->RefCount.increment(); + (*Event)->incrementRefCount(); const auto &ZeCommandList = CommandList->first; @@ -1635,14 +1635,14 @@ ur_result_t urMemBufferCreate( ur_result_t urMemRetain( /// [in] handle of the memory object to get access ur_mem_handle_t Mem) { - Mem->RefCount.increment(); + Mem->incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t urMemRelease( /// [in] handle of the memory object to release ur_mem_handle_t Mem) { - if (!Mem->RefCount.decrementAndTest()) + if (!Mem->decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (Mem->isImage()) { @@ -1848,7 +1848,7 @@ ur_result_t urMemGetInfo( return ReturnValue(size_t{Buffer->Size}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(Buffer->RefCount.load()); + return ReturnValue(Buffer->getRefCount()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index 715b5b51870c1..60c9609e750b7 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -116,7 +116,7 @@ struct ur_buffer final : ur_mem_handle_t_ { : ur_mem_handle_t_(mem_type_t::buffer, Parent->UrContext), Size(Size), SubBuffer{{Parent, Origin}} { // Retain the Parent Buffer due to the Creation of the SubBuffer. - Parent->RefCount.increment(); + Parent->incrementRefCount(); } // Interop-buffer constructor diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index 5d4d0acce0eb3..1ea177bcfab3f 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -42,12 +42,12 @@ ur_result_t urPhysicalMemCreate( } ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->RefCount.increment(); + hPhysicalMem->incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (!hPhysicalMem->RefCount.decrementAndTest()) + if (!hPhysicalMem->decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { @@ -68,7 +68,7 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, switch (propName) { case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hPhysicalMem->RefCount.load()); + return ReturnValue(hPhysicalMem->getRefCount()); } default: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; diff --git a/unified-runtime/source/adapters/level_zero/program.cpp b/unified-runtime/source/adapters/level_zero/program.cpp index 497e3057b7b9b..a9ae0fe10755e 100644 --- a/unified-runtime/source/adapters/level_zero/program.cpp +++ b/unified-runtime/source/adapters/level_zero/program.cpp @@ -558,14 +558,14 @@ ur_result_t urProgramLinkExp( ur_result_t urProgramRetain( /// [in] handle for the Program to retain ur_program_handle_t Program) { - Program->RefCount.increment(); + Program->incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t urProgramRelease( /// [in] handle for the Program to release ur_program_handle_t Program) { - if (!Program->RefCount.decrementAndTest()) + if (!Program->decrementRefCount() == 0) return UR_RESULT_SUCCESS; delete Program; @@ -708,7 +708,7 @@ ur_result_t urProgramGetInfo( switch (PropName) { case UR_PROGRAM_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Program->RefCount.load()}); + return ReturnValue(uint32_t{Program->getRefCount()}); case UR_PROGRAM_INFO_CONTEXT: return ReturnValue(Program->Context); case UR_PROGRAM_INFO_NUM_DEVICES: @@ -1115,7 +1115,7 @@ void ur_program_handle_t_::ur_release_program_resources(bool deletion) { // must be destroyed before the Module can be destroyed. So, be sure // to destroy build log before destroying the module. if (!deletion) { - if (!RefCount.decrementAndTest()) { + if (!decrementRefCount() == 0) { return; } } diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index dc7d19ffaa007..983d0ae72802f 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -369,7 +369,7 @@ ur_result_t urQueueGetInfo( case UR_QUEUE_INFO_DEVICE: return ReturnValue(Queue->Device); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Queue->RefCount.load()}); + return ReturnValue(uint32_t{Queue->getRefCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(Queue->Properties); case UR_QUEUE_INFO_SIZE: @@ -593,7 +593,7 @@ ur_result_t urQueueRetain( std::scoped_lock Lock(Queue->Mutex); Queue->RefCountExternal++; } - Queue->RefCount.increment(); + Queue->incrementRefCount(); return UR_RESULT_SUCCESS; } @@ -612,7 +612,7 @@ ur_result_t urQueueRelease( // internal reference count. When the External Reference count == 0, then // cleanup of the queue begins and the final decrement of the internal // reference count is completed. - static_cast(Queue->RefCount.decrementAndTest()); + static_cast(Queue->decrementRefCount() == 0); return UR_RESULT_SUCCESS; } @@ -1389,7 +1389,7 @@ ur_queue_handle_t_::executeCommandList(ur_command_list_ptr_t CommandList, if (!Event->HostVisibleEvent) { Event->HostVisibleEvent = reinterpret_cast(HostVisibleEvent); - HostVisibleEvent->RefCount.increment(); + HostVisibleEvent->incrementRefCount(); } } @@ -1550,7 +1550,7 @@ ur_result_t ur_queue_handle_t_::addEventToQueueCache(ur_event_handle_t Event) { } void ur_queue_handle_t_::active_barriers::add(ur_event_handle_t &Event) { - Event->RefCount.increment(); + Event->incrementRefCount(); Events.push_back(Event); } @@ -1588,7 +1588,7 @@ void ur_queue_handle_t_::clearEndTimeRecordings() { } ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) { - if (!Queue->RefCount.decrementAndTest()) + if (!Queue->decrementRefCount() == 0) return UR_RESULT_SUCCESS; for (auto &Cache : Queue->EventCaches) { @@ -1921,7 +1921,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // Append this Event to the CommandList, if any if (CommandList != Queue->CommandListMap.end()) { CommandList->second.append(*Event); - (*Event)->RefCount.increment(); + (*Event)->incrementRefCount(); } // We need to increment the reference counter here to avoid ur_queue_handle_t @@ -1929,7 +1929,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // urEventRelease requires access to the associated ur_queue_handle_t. // In urEventRelease, the reference counter of the Queue is decremented // to release it. - Queue->RefCount.increment(); + Queue->incrementRefCount(); // SYCL RT does not track completion of the events, so it could // release a PI event as soon as that's not being waited in the app. @@ -1961,7 +1961,7 @@ void ur_queue_handle_t_::CaptureIndirectAccesses() { // SubmissionsCount turns to 0. We don't want to know how many times // allocation was retained by each submission. if (Pair.second) - Elem.second.RefCount.increment(); + Elem.second.incrementRefCount(); } } Kernel->SubmissionsCount++; diff --git a/unified-runtime/source/adapters/level_zero/sampler.cpp b/unified-runtime/source/adapters/level_zero/sampler.cpp index 4f6f5760faada..2dea0d7e9bf65 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.cpp +++ b/unified-runtime/source/adapters/level_zero/sampler.cpp @@ -124,14 +124,14 @@ ur_result_t urSamplerCreate( ur_result_t urSamplerRetain( /// [in] handle of the sampler object to get access ur_sampler_handle_t Sampler) { - Sampler->RefCount.increment(); + Sampler->incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t urSamplerRelease( /// [in] handle of the sampler object to release ur_sampler_handle_t Sampler) { - if (!Sampler->RefCount.decrementAndTest()) + if (!Sampler->decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/usm.cpp b/unified-runtime/source/adapters/level_zero/usm.cpp index ab8556dd692c7..14ff5bdf95532 100644 --- a/unified-runtime/source/adapters/level_zero/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/usm.cpp @@ -523,14 +523,14 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t Pool) { - Pool->RefCount.increment(); + Pool->incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t Pool) { - if (Pool->RefCount.decrementAndTest()) { + if (Pool->decrementRefCount() == 0) { std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); delete Pool; @@ -553,7 +553,7 @@ ur_result_t urUSMPoolGetInfo( switch (PropName) { case UR_USM_POOL_INFO_REFERENCE_COUNT: { - return ReturnValue(Pool->RefCount.load()); + return ReturnValue(Pool->getRefCount()); } case UR_USM_POOL_INFO_CONTEXT: { return ReturnValue(Pool->Context); @@ -1250,7 +1250,7 @@ ur_result_t ZeMemFreeHelper(ur_context_handle_t Context, void *Ptr) { if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.RefCount.decrementAndTest()) { + if (!It->second.decrementRefCount() == 0) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } @@ -1297,7 +1297,7 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.RefCount.decrementAndTest()) { + if (!It->second.decrementRefCount() == 0) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp index cc2a88fb409e5..589e51d766526 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp @@ -258,7 +258,7 @@ urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - hCommandBuffer->RefCount.increment(); + hCommandBuffer->incrementRefCount(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -266,7 +266,7 @@ urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - if (!hCommandBuffer->RefCount.decrementAndTest()) + if (!hCommandBuffer->decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (auto executionEvent = hCommandBuffer->getExecutionEventUnlocked()) { @@ -630,7 +630,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hCommandBuffer->RefCount.load()}); + return ReturnValue(uint32_t{hCommandBuffer->getRefCount()}); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index d29496ec39425..326c1c9677fc3 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -916,7 +916,7 @@ ur_result_t ur_command_list_manager::appendNativeCommandExp( void ur_command_list_manager::recordSubmittedKernel( ur_kernel_handle_t hKernel) { submittedKernels.push_back(hKernel); - hKernel->RefCount.increment(); + hKernel->incrementRefCount(); } ze_command_list_handle_t ur_command_list_manager::getZeCommandList() { diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index 9ad4a4e61d633..735ad2a5cc3bb 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -80,12 +80,12 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext, defaultUSMPool(this, nullptr), asyncPool(this, nullptr) {} ur_result_t ur_context_handle_t_::retain() { - RefCount.increment(); + incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t ur_context_handle_t_::release() { - if (!RefCount.decrementAndTest()) + if (!decrementRefCount() == 0) return UR_RESULT_SUCCESS; delete this; @@ -201,7 +201,7 @@ ur_result_t urContextGetInfo(ur_context_handle_t hContext, case UR_CONTEXT_INFO_NUM_DEVICES: return ReturnValue(uint32_t(hContext->getDevices().size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hContext->RefCount.load()}); + return ReturnValue(uint32_t{hContext->getRefCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: // TODO: this is currently not implemented return ReturnValue(uint8_t{false}); diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index 30816a9fcdd6f..b839a34386f90 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -160,12 +160,12 @@ ze_event_handle_t ur_event_handle_t_::getZeEvent() const { } ur_result_t ur_event_handle_t_::retain() { - RefCount.increment(); + incrementRefCount(); return UR_RESULT_SUCCESS; } ur_result_t ur_event_handle_t_::release() { - if (!RefCount.decrementAndTest()) + if (!decrementRefCount() == 0) return UR_RESULT_SUCCESS; if (event_pool) { @@ -258,7 +258,7 @@ ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, } } case UR_EVENT_INFO_REFERENCE_COUNT: { - return returnValue(hEvent->RefCount.load()); + return returnValue(hEvent->getRefCount()); } case UR_EVENT_INFO_COMMAND_QUEUE: { auto urQueueHandle = reinterpret_cast(hEvent->getQueue()) - diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index 99d4852f9ad4a..a49f680999b84 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -52,8 +52,8 @@ void event_pool::free(ur_event_handle_t event) { freelist.push_back(event); // The event is still in the pool, so we need to increment the refcount - assert(event->RefCount.load() == 0); - event->RefCount.increment(); + assert(event->getRefCount() == 0); + event->incrementRefCount(); } event_provider *event_pool::getProvider() const { return provider.get(); } diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index a2189b57536e8..d214d87b4ab01 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -97,7 +97,7 @@ ur_kernel_handle_t_::ur_kernel_handle_t_( } ur_result_t ur_kernel_handle_t_::release() { - if (!RefCount.decrementAndTest()) + if (!decrementRefCount() == 0) return UR_RESULT_SUCCESS; // manually release kernels to allow errors to be propagated @@ -370,7 +370,7 @@ urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t hKernel) try { - hKernel->RefCount.increment(); + hKernel->incrementRefCount(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -634,7 +634,7 @@ ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, spills.size()); } case UR_KERNEL_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hKernel->RefCount.load()}); + return ReturnValue(uint32_t{hKernel->getRefCount()}); case UR_KERNEL_INFO_ATTRIBUTES: { auto attributes = hKernel->getSourceAttributes(); return ReturnValue(static_cast(attributes.data())); diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index b1f3829dd6967..b736ced309f73 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -671,7 +671,7 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, return returnValue(size_t{hMem->getBuffer()->getSize()}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return returnValue(hMem->getObject()->RefCount.load()); + return returnValue(hMem->getObject()->getRefCount()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -684,14 +684,14 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, } ur_result_t urMemRetain(ur_mem_handle_t hMem) try { - hMem->getObject()->RefCount.increment(); + hMem->getObject()->incrementRefCount(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urMemRelease(ur_mem_handle_t hMem) try { - if (!hMem->getObject()->RefCount.decrementAndTest()) + if (!hMem->getObject()->decrementRefCount() == 0) return UR_RESULT_SUCCESS; delete hMem; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp index 9831afdbc9e4c..88cd9fa8e4a88 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp @@ -50,7 +50,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRetain() { return std::visit( [](auto &q) { - q.RefCount.increment(); + q.incrementRefCount(); return UR_RESULT_SUCCESS; }, queue_data); @@ -59,7 +59,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRelease() { return std::visit( [queueHandle = this](auto &q) { - if (!q.RefCount.decrementAndTest()) + if (!q.decrementRefCount() == 0) return UR_RESULT_SUCCESS; delete queueHandle; return UR_RESULT_SUCCESS; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index cc9b464333e70..d7f1b0ebf9404 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -60,7 +60,7 @@ ur_queue_immediate_in_order_t::queueGetInfo(ur_queue_info_t propName, case UR_QUEUE_INFO_DEVICE: return ReturnValue(hDevice); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{RefCount.load()}); + return ReturnValue(uint32_t{getRefCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(flags); case UR_QUEUE_INFO_SIZE: diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index f455fd3763554..1ed8058ca0471 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -332,7 +332,7 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { - hPool->RefCount.increment(); + hPool->incrementRefCount(); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { return umf::umf2urResult(e); @@ -343,7 +343,7 @@ urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { - if (hPool->RefCount.decrementAndTest()) { + if (hPool->decrementRefCount() == 0) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; } @@ -369,7 +369,7 @@ ur_result_t urUSMPoolGetInfo( switch (propName) { case UR_USM_POOL_INFO_REFERENCE_COUNT: { - return ReturnValue(hPool->RefCount.load()); + return ReturnValue(hPool->getRefCount()); } case UR_USM_POOL_INFO_CONTEXT: { return ReturnValue(hPool->getContextHandle()); diff --git a/unified-runtime/source/ur/ur.hpp b/unified-runtime/source/ur/ur.hpp index 5b0914868777a..480fd16c769f4 100644 --- a/unified-runtime/source/ur/ur.hpp +++ b/unified-runtime/source/ur/ur.hpp @@ -228,6 +228,14 @@ template struct handle_base { // Handles are non-copyable. handle_base(const handle_base &) = delete; handle_base &operator=(const handle_base &) = delete; + + uint32_t getRefCount() const noexcept { return Count.load(); } + uint32_t incrementRefCount() { return ++Count; } + uint32_t decrementRefCount() { return --Count; } + void resetRefCount() { Count = 1; } + +private: + std::atomic_uint32_t Count{1}; }; template From 0cfdbee77c735977ae52b7556ef9a0561776e784 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Thu, 19 Jun 2025 10:19:10 +0100 Subject: [PATCH 2/7] Add decrementAndTest back in to fix logic in checks. --- .../source/adapters/level_zero/command_buffer.cpp | 2 +- unified-runtime/source/adapters/level_zero/context.cpp | 2 +- unified-runtime/source/adapters/level_zero/device.cpp | 2 +- unified-runtime/source/adapters/level_zero/event.cpp | 2 +- unified-runtime/source/adapters/level_zero/kernel.cpp | 2 +- unified-runtime/source/adapters/level_zero/memory.cpp | 2 +- unified-runtime/source/adapters/level_zero/physical_mem.cpp | 2 +- unified-runtime/source/adapters/level_zero/program.cpp | 4 ++-- unified-runtime/source/adapters/level_zero/queue.cpp | 4 ++-- unified-runtime/source/adapters/level_zero/sampler.cpp | 2 +- unified-runtime/source/adapters/level_zero/usm.cpp | 6 +++--- .../source/adapters/level_zero/v2/command_buffer.cpp | 2 +- unified-runtime/source/adapters/level_zero/v2/context.cpp | 2 +- unified-runtime/source/adapters/level_zero/v2/event.cpp | 2 +- unified-runtime/source/adapters/level_zero/v2/kernel.cpp | 2 +- unified-runtime/source/adapters/level_zero/v2/memory.cpp | 2 +- .../source/adapters/level_zero/v2/queue_handle.hpp | 2 +- unified-runtime/source/adapters/level_zero/v2/usm.cpp | 2 +- unified-runtime/source/ur/ur.hpp | 1 + 19 files changed, 23 insertions(+), 22 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index 108093764c1cd..cdd9f73b68421 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -846,7 +846,7 @@ urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) { ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) { - if (!CommandBuffer->decrementRefCount() == 0) + if (!CommandBuffer->decrementAndTest()) return UR_RESULT_SUCCESS; UR_CALL(waitForOngoingExecution(CommandBuffer)); diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index 6c9553001ce5d..5ddd2170a3177 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -251,7 +251,7 @@ ur_device_handle_t ur_context_handle_t_::getRootDevice() const { // from the list of tracked contexts. ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { - if (!Context->decrementRefCount() == 0) + if (!Context->decrementAndTest()) return UR_RESULT_SUCCESS; if (IndirectAccessTrackingEnabled) { diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 0b85b5845da01..bf6e8dd9b7a42 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1674,7 +1674,7 @@ ur_result_t urDeviceRetain(ur_device_handle_t Device) { ur_result_t urDeviceRelease(ur_device_handle_t Device) { // Root devices are destroyed during the piTearDown process. if (Device->isSubDevice()) { - if (Device->decrementRefCount() == 0) { + if (Device->decrementAndTest()) { delete Device; } } diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index 2ff41ea6dc968..06ac07a7201ee 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -1088,7 +1088,7 @@ ur_event_handle_t_::~ur_event_handle_t_() { ur_result_t urEventReleaseInternal(ur_event_handle_t Event, bool *isEventDeleted) { - if (!Event->decrementRefCount() == 0) + if (!Event->decrementAndTest()) return UR_RESULT_SUCCESS; if (Event->OriginAllocEvent) { diff --git a/unified-runtime/source/adapters/level_zero/kernel.cpp b/unified-runtime/source/adapters/level_zero/kernel.cpp index 64c4594759da9..d7e95b446b6b8 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/kernel.cpp @@ -946,7 +946,7 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release ur_kernel_handle_t Kernel) { - if (!Kernel->decrementRefCount() == 0) + if (!Kernel->decrementAndTest()) return UR_RESULT_SUCCESS; auto KernelProgram = Kernel->Program; diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index af17d325173cd..77142314a2493 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -1642,7 +1642,7 @@ ur_result_t urMemRetain( ur_result_t urMemRelease( /// [in] handle of the memory object to release ur_mem_handle_t Mem) { - if (!Mem->decrementRefCount() == 0) + if (!Mem->decrementAndTest()) return UR_RESULT_SUCCESS; if (Mem->isImage()) { diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index 1ea177bcfab3f..570a6b27268ff 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -47,7 +47,7 @@ ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { } ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (!hPhysicalMem->decrementRefCount() == 0) + if (!hPhysicalMem->decrementAndTest()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/program.cpp b/unified-runtime/source/adapters/level_zero/program.cpp index a9ae0fe10755e..427aaf33a20d6 100644 --- a/unified-runtime/source/adapters/level_zero/program.cpp +++ b/unified-runtime/source/adapters/level_zero/program.cpp @@ -565,7 +565,7 @@ ur_result_t urProgramRetain( ur_result_t urProgramRelease( /// [in] handle for the Program to release ur_program_handle_t Program) { - if (!Program->decrementRefCount() == 0) + if (!Program->decrementAndTest()) return UR_RESULT_SUCCESS; delete Program; @@ -1115,7 +1115,7 @@ void ur_program_handle_t_::ur_release_program_resources(bool deletion) { // must be destroyed before the Module can be destroyed. So, be sure // to destroy build log before destroying the module. if (!deletion) { - if (!decrementRefCount() == 0) { + if (!decrementAndTest()) { return; } } diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index 983d0ae72802f..203c516df1886 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -612,7 +612,7 @@ ur_result_t urQueueRelease( // internal reference count. When the External Reference count == 0, then // cleanup of the queue begins and the final decrement of the internal // reference count is completed. - static_cast(Queue->decrementRefCount() == 0); + static_cast(Queue->decrementAndTest()); return UR_RESULT_SUCCESS; } @@ -1588,7 +1588,7 @@ void ur_queue_handle_t_::clearEndTimeRecordings() { } ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) { - if (!Queue->decrementRefCount() == 0) + if (!Queue->decrementAndTest()) return UR_RESULT_SUCCESS; for (auto &Cache : Queue->EventCaches) { diff --git a/unified-runtime/source/adapters/level_zero/sampler.cpp b/unified-runtime/source/adapters/level_zero/sampler.cpp index 2dea0d7e9bf65..433043f11ae9a 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.cpp +++ b/unified-runtime/source/adapters/level_zero/sampler.cpp @@ -131,7 +131,7 @@ ur_result_t urSamplerRetain( ur_result_t urSamplerRelease( /// [in] handle of the sampler object to release ur_sampler_handle_t Sampler) { - if (!Sampler->decrementRefCount() == 0) + if (!Sampler->decrementAndTest()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/usm.cpp b/unified-runtime/source/adapters/level_zero/usm.cpp index 14ff5bdf95532..aa7a4c852f996 100644 --- a/unified-runtime/source/adapters/level_zero/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/usm.cpp @@ -530,7 +530,7 @@ urUSMPoolRetain(ur_usm_pool_handle_t Pool) { ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t Pool) { - if (Pool->decrementRefCount() == 0) { + if (Pool->decrementAndTest()) { std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); delete Pool; @@ -1250,7 +1250,7 @@ ur_result_t ZeMemFreeHelper(ur_context_handle_t Context, void *Ptr) { if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.decrementRefCount() == 0) { + if (!It->second.decrementAndTest()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } @@ -1297,7 +1297,7 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.decrementRefCount() == 0) { + if (!It->second.decrementAndTest()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp index 589e51d766526..2e5d74868cbaa 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp @@ -266,7 +266,7 @@ urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - if (!hCommandBuffer->decrementRefCount() == 0) + if (!hCommandBuffer->decrementAndTest()) return UR_RESULT_SUCCESS; if (auto executionEvent = hCommandBuffer->getExecutionEventUnlocked()) { diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index 735ad2a5cc3bb..f5b37651f8542 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -85,7 +85,7 @@ ur_result_t ur_context_handle_t_::retain() { } ur_result_t ur_context_handle_t_::release() { - if (!decrementRefCount() == 0) + if (!decrementAndTest()) return UR_RESULT_SUCCESS; delete this; diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index b839a34386f90..bc520a09a0b66 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -165,7 +165,7 @@ ur_result_t ur_event_handle_t_::retain() { } ur_result_t ur_event_handle_t_::release() { - if (!decrementRefCount() == 0) + if (!decrementAndTest()) return UR_RESULT_SUCCESS; if (event_pool) { diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index d214d87b4ab01..1ae6a7e8528a2 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -97,7 +97,7 @@ ur_kernel_handle_t_::ur_kernel_handle_t_( } ur_result_t ur_kernel_handle_t_::release() { - if (!decrementRefCount() == 0) + if (!decrementAndTest()) return UR_RESULT_SUCCESS; // manually release kernels to allow errors to be propagated diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index b736ced309f73..49eba95fc25d2 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -691,7 +691,7 @@ ur_result_t urMemRetain(ur_mem_handle_t hMem) try { } ur_result_t urMemRelease(ur_mem_handle_t hMem) try { - if (!hMem->getObject()->decrementRefCount() == 0) + if (!hMem->getObject()->decrementAndTest()) return UR_RESULT_SUCCESS; delete hMem; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp index 88cd9fa8e4a88..94dd024b2a667 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp @@ -59,7 +59,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRelease() { return std::visit( [queueHandle = this](auto &q) { - if (!q.decrementRefCount() == 0) + if (!q.decrementAndTest()) return UR_RESULT_SUCCESS; delete queueHandle; return UR_RESULT_SUCCESS; diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index 1ed8058ca0471..84bac0db1b894 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -343,7 +343,7 @@ urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { - if (hPool->decrementRefCount() == 0) { + if (hPool->decrementAndTest()) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; } diff --git a/unified-runtime/source/ur/ur.hpp b/unified-runtime/source/ur/ur.hpp index 480fd16c769f4..7331ebc4742f5 100644 --- a/unified-runtime/source/ur/ur.hpp +++ b/unified-runtime/source/ur/ur.hpp @@ -232,6 +232,7 @@ template struct handle_base { uint32_t getRefCount() const noexcept { return Count.load(); } uint32_t incrementRefCount() { return ++Count; } uint32_t decrementRefCount() { return --Count; } + bool decrementAndTest() { return --Count == 0; } void resetRefCount() { Count = 1; } private: From 7caa4cda4d02f0b2a664ad063a92beb51f23bc55 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Fri, 20 Jun 2025 14:33:03 +0100 Subject: [PATCH 3/7] change resetRefCount to have a param and a default value so l0 adapter can start ref count from 0 --- unified-runtime/source/adapters/level_zero/adapter.cpp | 2 ++ unified-runtime/source/ur/ur.hpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 25edec5417df9..2663c4c44bb63 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -301,6 +301,8 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; + resetRefCount(0); + #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's // destruction of its context until after the UR Adapter is destroyed. diff --git a/unified-runtime/source/ur/ur.hpp b/unified-runtime/source/ur/ur.hpp index 7331ebc4742f5..ec41529d9f575 100644 --- a/unified-runtime/source/ur/ur.hpp +++ b/unified-runtime/source/ur/ur.hpp @@ -233,7 +233,7 @@ template struct handle_base { uint32_t incrementRefCount() { return ++Count; } uint32_t decrementRefCount() { return --Count; } bool decrementAndTest() { return --Count == 0; } - void resetRefCount() { Count = 1; } + void resetRefCount(uint32_t value = 1) { Count = value; } private: std::atomic_uint32_t Count{1}; From c2782cb3c98d63e1c2954545613bd62ce53c2d39 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Wed, 25 Jun 2025 12:17:06 +0100 Subject: [PATCH 4/7] Change implementation to a new URRefCount class and add it as member to any relevant handles that need ref counting. --- .../source/adapters/level_zero/adapter.cpp | 14 ++++----- .../source/adapters/level_zero/adapter.hpp | 6 ++++ .../adapters/level_zero/async_alloc.cpp | 2 +- .../adapters/level_zero/command_buffer.cpp | 10 +++---- .../adapters/level_zero/command_buffer.hpp | 6 ++++ .../source/adapters/level_zero/common.hpp | 6 ++++ .../source/adapters/level_zero/context.cpp | 6 ++-- .../source/adapters/level_zero/context.hpp | 5 ++++ .../source/adapters/level_zero/device.cpp | 6 ++-- .../source/adapters/level_zero/device.hpp | 6 ++++ .../source/adapters/level_zero/event.cpp | 16 +++++----- .../source/adapters/level_zero/event.hpp | 6 ++++ .../source/adapters/level_zero/kernel.cpp | 6 ++-- .../source/adapters/level_zero/kernel.hpp | 9 +++++- .../source/adapters/level_zero/memory.cpp | 10 +++---- .../source/adapters/level_zero/memory.hpp | 8 ++++- .../adapters/level_zero/physical_mem.cpp | 6 ++-- .../adapters/level_zero/physical_mem.hpp | 6 ++++ .../source/adapters/level_zero/program.cpp | 8 ++--- .../source/adapters/level_zero/program.hpp | 5 ++++ .../source/adapters/level_zero/queue.cpp | 18 ++++++------ .../source/adapters/level_zero/queue.hpp | 6 ++++ .../source/adapters/level_zero/sampler.cpp | 4 +-- .../source/adapters/level_zero/sampler.hpp | 6 ++++ .../source/adapters/level_zero/usm.cpp | 10 +++---- .../source/adapters/level_zero/usm.hpp | 9 ++++-- .../adapters/level_zero/v2/command_buffer.cpp | 6 ++-- .../adapters/level_zero/v2/command_buffer.hpp | 9 +++++- .../level_zero/v2/command_list_manager.cpp | 2 +- .../source/adapters/level_zero/v2/context.cpp | 6 ++-- .../source/adapters/level_zero/v2/context.hpp | 5 ++++ .../source/adapters/level_zero/v2/event.cpp | 6 ++-- .../source/adapters/level_zero/v2/event.hpp | 5 ++++ .../adapters/level_zero/v2/event_pool.cpp | 4 +-- .../source/adapters/level_zero/v2/kernel.cpp | 6 ++-- .../source/adapters/level_zero/v2/kernel.hpp | 5 ++++ .../source/adapters/level_zero/v2/memory.cpp | 6 ++-- .../source/adapters/level_zero/v2/memory.hpp | 14 ++++----- .../adapters/level_zero/v2/queue_handle.hpp | 7 +++-- .../v2/queue_immediate_in_order.cpp | 2 +- .../v2/queue_immediate_in_order.hpp | 4 +++ .../source/adapters/level_zero/v2/usm.cpp | 6 ++-- .../source/adapters/level_zero/v2/usm.hpp | 5 ++++ .../source/common/ur_ref_count.hpp | 29 +++++++++++++++++++ unified-runtime/source/ur/ur.hpp | 9 ------ 45 files changed, 231 insertions(+), 105 deletions(-) create mode 100644 unified-runtime/source/common/ur_ref_count.hpp diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 2663c4c44bb63..899393553643b 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -301,7 +301,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; - resetRefCount(0); + RefCount.reset(0); #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's @@ -677,7 +677,7 @@ ur_result_t urAdapterGet( } *Adapters = GlobalAdapter; - if (GlobalAdapter->incrementRefCount() == 0) { + if (GlobalAdapter->getRefCount().increment() == 0) { adapterStateInit(); } } @@ -694,7 +694,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { // NOTE: This does not require guarding with a mutex; the instant the ref // count hits zero, both Get and Retain are UB. - if (GlobalAdapter->decrementRefCount() == 0) { + if (GlobalAdapter->getRefCount().decrementAndTest()) { auto result = adapterStateTeardown(); #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's @@ -711,9 +711,9 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { return UR_RESULT_SUCCESS; } -ur_result_t urAdapterRetain(ur_adapter_handle_t) { +ur_result_t urAdapterRetain([[maybe_unused]] ur_adapter_handle_t Adapter) { assert(GlobalAdapter && GlobalAdapter == Adapter); - GlobalAdapter->incrementRefCount(); + GlobalAdapter->getRefCount().increment(); return UR_RESULT_SUCCESS; } @@ -742,12 +742,12 @@ ur_result_t urAdapterGetInfo(ur_adapter_handle_t, ur_adapter_info_t PropName, case UR_ADAPTER_INFO_BACKEND: return ReturnValue(UR_BACKEND_LEVEL_ZERO); case UR_ADAPTER_INFO_REFERENCE_COUNT: - return ReturnValue(GlobalAdapter->getRefCount()); + return ReturnValue(GlobalAdapter->getRefCount().getCount()); case UR_ADAPTER_INFO_VERSION: { #ifdef UR_ADAPTER_LEVEL_ZERO_V2 uint32_t adapterVersion = 2; #else - uint32_t adapterVersion = 1; + uint32_t adapterVersion = 1; #endif return ReturnValue(adapterVersion); } diff --git a/unified-runtime/source/adapters/level_zero/adapter.hpp b/unified-runtime/source/adapters/level_zero/adapter.hpp index 59fd0971cd20f..c75921c57130f 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.hpp +++ b/unified-runtime/source/adapters/level_zero/adapter.hpp @@ -9,6 +9,7 @@ //===----------------------------------------------------------------------===// #pragma once +#include "common/ur_ref_count.hpp" #include "logger/ur_logger.hpp" #include "ur_interface_loader.hpp" #include @@ -43,6 +44,11 @@ struct ur_adapter_handle_t_ : ur::handle_base { ZeCache> PlatformCache; logger::Logger &logger; HMODULE processHandle = nullptr; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; extern ur_adapter_handle_t_ *GlobalAdapter; diff --git a/unified-runtime/source/adapters/level_zero/async_alloc.cpp b/unified-runtime/source/adapters/level_zero/async_alloc.cpp index 96729c9da0f5d..d29bd6fd64ab6 100644 --- a/unified-runtime/source/adapters/level_zero/async_alloc.cpp +++ b/unified-runtime/source/adapters/level_zero/async_alloc.cpp @@ -247,7 +247,7 @@ ur_result_t urEnqueueUSMFreeExp( } size_t size = umfPoolMallocUsableSize(hPool, Mem); - (*Event)->incrementRefCount(); + (*Event)->getRefCount().increment(); usmPool->AsyncPool.insert(Mem, size, *Event, Queue); // Signal that USM free event was finished diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index cdd9f73b68421..939779ef9cb44 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -840,13 +840,13 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) { - CommandBuffer->incrementRefCount(); + CommandBuffer->getRefCount().increment(); return UR_RESULT_SUCCESS; } ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) { - if (!CommandBuffer->decrementAndTest()) + if (!CommandBuffer->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; UR_CALL(waitForOngoingExecution(CommandBuffer)); @@ -1641,7 +1641,7 @@ ur_result_t enqueueImmediateAppendPath( if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->incrementRefCount(); + (*Event)->getRefCount().increment(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(CommandListHelper, false, false)); @@ -1724,7 +1724,7 @@ ur_result_t enqueueWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer, if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->incrementRefCount(); + (*Event)->getRefCount().increment(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(SignalCommandList, false /*IsBlocking*/, @@ -1848,7 +1848,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hCommandBuffer->getRefCount()}); + return ReturnValue(uint32_t{hCommandBuffer->getRefCount().getCount()}); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/command_buffer.hpp index f7b62a9c8dd1e..31f9bf5b114ab 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.hpp @@ -17,6 +17,7 @@ #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "context.hpp" #include "kernel.hpp" #include "queue.hpp" @@ -149,4 +150,9 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { // Track handle objects to free when command-buffer is destroyed. std::vector> CommandHandles; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index 7702332e3d8de..a94b83a891332 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -34,6 +34,7 @@ #include #include +#include "common/ur_ref_count.hpp" #include "logger/ur_logger.hpp" #include "ur_interface_loader.hpp" @@ -257,6 +258,11 @@ struct MemAllocRecord : ur_object { // TODO: this should go away when memory isolation issue is fixed in the Level // Zero runtime. ur_context_handle_t Context; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; extern usm::DisjointPoolAllConfigs DisjointPoolConfigInstance; diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index 5ddd2170a3177..a6f9450fefd9e 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -61,7 +61,7 @@ ur_result_t urContextRetain( /// [in] handle of the context to get a reference of. ur_context_handle_t Context) { - Context->incrementRefCount(); + Context->getRefCount().increment(); return UR_RESULT_SUCCESS; } @@ -113,7 +113,7 @@ ur_result_t urContextGetInfo( case UR_CONTEXT_INFO_NUM_DEVICES: return ReturnValue(uint32_t(Context->Devices.size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Context->getRefCount()}); + return ReturnValue(uint32_t{Context->getRefCount().getCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: // 2D USM memcpy is supported. return ReturnValue(uint8_t{UseMemcpy2DOperations}); @@ -251,7 +251,7 @@ ur_device_handle_t ur_context_handle_t_::getRootDevice() const { // from the list of tracked contexts. ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { - if (!Context->decrementAndTest()) + if (!Context->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; if (IndirectAccessTrackingEnabled) { diff --git a/unified-runtime/source/adapters/level_zero/context.hpp b/unified-runtime/source/adapters/level_zero/context.hpp index 86e0ea27b5c3e..70f9f1e3ae044 100644 --- a/unified-runtime/source/adapters/level_zero/context.hpp +++ b/unified-runtime/source/adapters/level_zero/context.hpp @@ -26,6 +26,7 @@ #include "queue.hpp" #include "usm.hpp" +#include "common/ur_ref_count.hpp" #include struct l0_command_list_cache_info { @@ -358,6 +359,8 @@ struct ur_context_handle_t_ : ur_object { // Get handle to the L0 context ze_context_handle_t getZeHandle() const; + URRefCount &getRefCount() noexcept { return RefCount; } + private: enum EventFlags { EVENT_FLAG_HOST_VISIBLE = UR_BIT(0), @@ -404,6 +407,8 @@ struct ur_context_handle_t_ : ur_object { return &EventCaches[index]; } + + URRefCount RefCount; }; // Helper function to release the context, a caller must lock the platform-level diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index bf6e8dd9b7a42..6c43b28b80760 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -470,7 +470,7 @@ ur_result_t urDeviceGetInfo( return ReturnValue((uint32_t)Device->SubDevices.size()); } case UR_DEVICE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Device->getRefCount()}); + return ReturnValue(uint32_t{Device->getRefCount().getCount()}); case UR_DEVICE_INFO_SUPPORTED_PARTITIONS: { // SYCL spec says: if this SYCL device cannot be partitioned into at least // two sub devices then the returned vector must be empty. @@ -1666,7 +1666,7 @@ ur_result_t urDeviceGetGlobalTimestamps( ur_result_t urDeviceRetain(ur_device_handle_t Device) { // The root-device ref-count remains unchanged (always 1). if (Device->isSubDevice()) { - Device->incrementRefCount(); + Device->getRefCount().increment(); } return UR_RESULT_SUCCESS; } @@ -1674,7 +1674,7 @@ ur_result_t urDeviceRetain(ur_device_handle_t Device) { ur_result_t urDeviceRelease(ur_device_handle_t Device) { // Root devices are destroyed during the piTearDown process. if (Device->isSubDevice()) { - if (Device->decrementAndTest()) { + if (Device->getRefCount().decrementAndTest()) { delete Device; } } diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index a8326c0cf668b..d52e240bc53b9 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -20,6 +20,7 @@ #include "adapters/level_zero/platform.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" #include #include #include @@ -242,6 +243,11 @@ struct ur_device_handle_t_ : ur_object { // unique ephemeral identifer of the device in the adapter std::optional Id; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; inline std::vector diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index 06ac07a7201ee..b043f36a707f2 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -505,7 +505,7 @@ ur_result_t urEventGetInfo( return ReturnValue(Result); } case UR_EVENT_INFO_REFERENCE_COUNT: { - return ReturnValue(Event->getRefCount()); + return ReturnValue(Event->getRefCount().getCount()); } default: UR_LOG(ERR, "Unsupported ParamName in urEventGetInfo: ParamName={}(0x{})", @@ -874,7 +874,7 @@ ur_result_t /// [in] handle of the event object urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { Event->RefCountExternal++; - Event->incrementRefCount(); + Event->getRefCount().increment(); return UR_RESULT_SUCCESS; } @@ -1088,7 +1088,7 @@ ur_event_handle_t_::~ur_event_handle_t_() { ur_result_t urEventReleaseInternal(ur_event_handle_t Event, bool *isEventDeleted) { - if (!Event->decrementAndTest()) + if (!Event->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; if (Event->OriginAllocEvent) { @@ -1429,7 +1429,7 @@ ur_result_t ur_event_handle_t_::reset() { CommandType = UR_EXT_COMMAND_TYPE_USER; WaitList = {}; RefCountExternal = 0; - resetRefCount(); + RefCount.reset(); CommandList = std::nullopt; completionBatch = std::nullopt; OriginAllocEvent = nullptr; @@ -1524,7 +1524,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( std::shared_lock Lock(CurQueue->LastCommandEvent->Mutex); this->ZeEventList[0] = CurQueue->LastCommandEvent->ZeEvent; this->UrEventList[0] = CurQueue->LastCommandEvent; - this->UrEventList[0]->incrementRefCount(); + this->UrEventList[0]->getRefCount().increment(); TmpListLength = 1; } else if (EventListLength > 0) { this->ZeEventList = new ze_event_handle_t[EventListLength]; @@ -1660,7 +1660,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( IsInternal, IsMultiDevice)); MultiDeviceZeEvent = MultiDeviceEvent->ZeEvent; const auto &ZeCommandList = CommandList->first; - EventList[I]->incrementRefCount(); + EventList[I]->getRefCount().increment(); // Append a Barrier to wait on the original event while signalling the // new multi device event. @@ -1676,11 +1676,11 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( this->ZeEventList[TmpListLength] = MultiDeviceZeEvent; this->UrEventList[TmpListLength] = MultiDeviceEvent; - this->UrEventList[TmpListLength]->incrementRefCount(); + this->UrEventList[TmpListLength]->getRefCount().increment(); } else { this->ZeEventList[TmpListLength] = EventList[I]->ZeEvent; this->UrEventList[TmpListLength] = EventList[I]; - this->UrEventList[TmpListLength]->incrementRefCount(); + this->UrEventList[TmpListLength]->getRefCount().increment(); } if (QueueLock.has_value()) { diff --git a/unified-runtime/source/adapters/level_zero/event.hpp b/unified-runtime/source/adapters/level_zero/event.hpp index 13b36bcdfbe94..51e187e29b72e 100644 --- a/unified-runtime/source/adapters/level_zero/event.hpp +++ b/unified-runtime/source/adapters/level_zero/event.hpp @@ -25,6 +25,7 @@ #include #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "queue.hpp" #include "ur_api.h" @@ -262,6 +263,11 @@ struct ur_event_handle_t_ : ur_object { // Used only for asynchronous allocations. This is the event originally used // on async free to indicate when the allocation can be used again. ur_event_handle_t OriginAllocEvent = nullptr; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; // Helper function to implement zeHostSynchronize. diff --git a/unified-runtime/source/adapters/level_zero/kernel.cpp b/unified-runtime/source/adapters/level_zero/kernel.cpp index d7e95b446b6b8..29889beb13c4e 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/kernel.cpp @@ -787,7 +787,7 @@ ur_result_t urKernelGetInfo( } } case UR_KERNEL_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Kernel->getRefCount()}); + return ReturnValue(uint32_t{Kernel->getRefCount().getCount()}); case UR_KERNEL_INFO_ATTRIBUTES: try { uint32_t Size; @@ -938,7 +938,7 @@ ur_result_t urKernelGetSubGroupInfo( ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t Kernel) { - Kernel->incrementRefCount(); + Kernel->getRefCount().increment(); return UR_RESULT_SUCCESS; } @@ -946,7 +946,7 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release ur_kernel_handle_t Kernel) { - if (!Kernel->decrementAndTest()) + if (!Kernel->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; auto KernelProgram = Kernel->Program; diff --git a/unified-runtime/source/adapters/level_zero/kernel.hpp b/unified-runtime/source/adapters/level_zero/kernel.hpp index 7f80348cda31f..0666edc7d8a27 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/kernel.hpp @@ -9,9 +9,11 @@ //===----------------------------------------------------------------------===// #pragma once +#include + #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "memory.hpp" -#include struct ur_kernel_handle_t_ : ur_object { ur_kernel_handle_t_(bool OwnZeHandle, ur_program_handle_t Program) @@ -106,6 +108,11 @@ struct ur_kernel_handle_t_ : ur_object { // Cache of the kernel properties. ZeCache> ZeKernelProperties; ZeCache ZeKernelName; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; ur_result_t getZeKernel(ze_device_handle_t hDevice, ur_kernel_handle_t hKernel, diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index 77142314a2493..c855b42e856c3 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -1052,7 +1052,7 @@ ur_result_t urEnqueueMemBufferMap( // Add the event to the command list. CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->incrementRefCount(); + (*Event)->getRefCount().increment(); const auto &ZeCommandList = CommandList->first; const auto &WaitList = (*Event)->WaitList; @@ -1183,7 +1183,7 @@ ur_result_t urEnqueueMemUnmap( nullptr /*ForcedCmdQueue*/)); CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->incrementRefCount(); + (*Event)->getRefCount().increment(); const auto &ZeCommandList = CommandList->first; @@ -1635,14 +1635,14 @@ ur_result_t urMemBufferCreate( ur_result_t urMemRetain( /// [in] handle of the memory object to get access ur_mem_handle_t Mem) { - Mem->incrementRefCount(); + Mem->getRefCount().increment(); return UR_RESULT_SUCCESS; } ur_result_t urMemRelease( /// [in] handle of the memory object to release ur_mem_handle_t Mem) { - if (!Mem->decrementAndTest()) + if (!Mem->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; if (Mem->isImage()) { @@ -1848,7 +1848,7 @@ ur_result_t urMemGetInfo( return ReturnValue(size_t{Buffer->Size}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(Buffer->getRefCount()); + return ReturnValue(Buffer->getRefCount().getCount()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index 60c9609e750b7..280a7fc262b1d 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -19,6 +19,7 @@ #include #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "context.hpp" #include "event.hpp" #include "program.hpp" @@ -90,6 +91,8 @@ struct ur_mem_handle_t_ : ur_object { // Method to get type of the derived object (image or buffer) bool isImage() const { return mem_type == mem_type_t::image; } + URRefCount &getRefCount() noexcept { return RefCount; } + protected: ur_mem_handle_t_(mem_type_t type, ur_context_handle_t Context) : UrContext{Context}, UrDevice{nullptr}, mem_type(type) {} @@ -101,6 +104,9 @@ struct ur_mem_handle_t_ : ur_object { // Since the destructor isn't virtual, callers must destruct it via ur_buffer // or ur_image ~ur_mem_handle_t_() {}; + +private: + URRefCount RefCount; }; struct ur_buffer final : ur_mem_handle_t_ { @@ -116,7 +122,7 @@ struct ur_buffer final : ur_mem_handle_t_ { : ur_mem_handle_t_(mem_type_t::buffer, Parent->UrContext), Size(Size), SubBuffer{{Parent, Origin}} { // Retain the Parent Buffer due to the Creation of the SubBuffer. - Parent->incrementRefCount(); + Parent->getRefCount().increment(); } // Interop-buffer constructor diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index 570a6b27268ff..480250a3af7ea 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -42,12 +42,12 @@ ur_result_t urPhysicalMemCreate( } ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->incrementRefCount(); + hPhysicalMem->getRefCount().increment(); return UR_RESULT_SUCCESS; } ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (!hPhysicalMem->decrementAndTest()) + if (!hPhysicalMem->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { @@ -68,7 +68,7 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, switch (propName) { case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hPhysicalMem->getRefCount()); + return ReturnValue(hPhysicalMem->getRefCount().getCount()); } default: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/physical_mem.hpp index 6ce630bcc5e1f..235253cef04f8 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.hpp @@ -10,6 +10,7 @@ #pragma once #include "common.hpp" +#include "common/ur_ref_count.hpp" struct ur_physical_mem_handle_t_ : ur_object { ur_physical_mem_handle_t_(ze_physical_mem_handle_t ZePhysicalMem, @@ -21,4 +22,9 @@ struct ur_physical_mem_handle_t_ : ur_object { // Keeps the PI context of this memory handle. ur_context_handle_t Context; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/program.cpp b/unified-runtime/source/adapters/level_zero/program.cpp index 427aaf33a20d6..921fa9fe991f0 100644 --- a/unified-runtime/source/adapters/level_zero/program.cpp +++ b/unified-runtime/source/adapters/level_zero/program.cpp @@ -558,14 +558,14 @@ ur_result_t urProgramLinkExp( ur_result_t urProgramRetain( /// [in] handle for the Program to retain ur_program_handle_t Program) { - Program->incrementRefCount(); + Program->getRefCount().increment(); return UR_RESULT_SUCCESS; } ur_result_t urProgramRelease( /// [in] handle for the Program to release ur_program_handle_t Program) { - if (!Program->decrementAndTest()) + if (!Program->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; delete Program; @@ -708,7 +708,7 @@ ur_result_t urProgramGetInfo( switch (PropName) { case UR_PROGRAM_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Program->getRefCount()}); + return ReturnValue(uint32_t{Program->getRefCount().getCount()}); case UR_PROGRAM_INFO_CONTEXT: return ReturnValue(Program->Context); case UR_PROGRAM_INFO_NUM_DEVICES: @@ -1115,7 +1115,7 @@ void ur_program_handle_t_::ur_release_program_resources(bool deletion) { // must be destroyed before the Module can be destroyed. So, be sure // to destroy build log before destroying the module. if (!deletion) { - if (!decrementAndTest()) { + if (!RefCount.decrementAndTest()) { return; } } diff --git a/unified-runtime/source/adapters/level_zero/program.hpp b/unified-runtime/source/adapters/level_zero/program.hpp index 789daf052ba0c..29193afbe5ef0 100644 --- a/unified-runtime/source/adapters/level_zero/program.hpp +++ b/unified-runtime/source/adapters/level_zero/program.hpp @@ -10,6 +10,7 @@ #pragma once #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "device.hpp" struct ur_program_handle_t_ : ur_object { @@ -226,6 +227,8 @@ struct ur_program_handle_t_ : ur_object { // UR_PROGRAM_INFO_BINARY_SIZES. const std::vector AssociatedDevices; + URRefCount &getRefCount() noexcept { return RefCount; } + private: struct DeviceData { // Log from the result of building the program for the device using @@ -264,4 +267,6 @@ struct ur_program_handle_t_ : ur_object { // handle from the program. // TODO: Currently interoparability UR API does not support multiple devices. ze_module_handle_t InteropZeModule = nullptr; + + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index 203c516df1886..b419dbc8f5443 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -369,7 +369,7 @@ ur_result_t urQueueGetInfo( case UR_QUEUE_INFO_DEVICE: return ReturnValue(Queue->Device); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Queue->getRefCount()}); + return ReturnValue(uint32_t{Queue->getRefCount().getCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(Queue->Properties); case UR_QUEUE_INFO_SIZE: @@ -593,7 +593,7 @@ ur_result_t urQueueRetain( std::scoped_lock Lock(Queue->Mutex); Queue->RefCountExternal++; } - Queue->incrementRefCount(); + Queue->getRefCount().increment(); return UR_RESULT_SUCCESS; } @@ -612,7 +612,7 @@ ur_result_t urQueueRelease( // internal reference count. When the External Reference count == 0, then // cleanup of the queue begins and the final decrement of the internal // reference count is completed. - static_cast(Queue->decrementAndTest()); + static_cast(Queue->getRefCount().decrementAndTest()); return UR_RESULT_SUCCESS; } @@ -1389,7 +1389,7 @@ ur_queue_handle_t_::executeCommandList(ur_command_list_ptr_t CommandList, if (!Event->HostVisibleEvent) { Event->HostVisibleEvent = reinterpret_cast(HostVisibleEvent); - HostVisibleEvent->incrementRefCount(); + HostVisibleEvent->getRefCount().increment(); } } @@ -1550,7 +1550,7 @@ ur_result_t ur_queue_handle_t_::addEventToQueueCache(ur_event_handle_t Event) { } void ur_queue_handle_t_::active_barriers::add(ur_event_handle_t &Event) { - Event->incrementRefCount(); + Event->getRefCount().increment(); Events.push_back(Event); } @@ -1588,7 +1588,7 @@ void ur_queue_handle_t_::clearEndTimeRecordings() { } ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) { - if (!Queue->decrementAndTest()) + if (!Queue->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; for (auto &Cache : Queue->EventCaches) { @@ -1921,7 +1921,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // Append this Event to the CommandList, if any if (CommandList != Queue->CommandListMap.end()) { CommandList->second.append(*Event); - (*Event)->incrementRefCount(); + (*Event)->getRefCount().increment(); } // We need to increment the reference counter here to avoid ur_queue_handle_t @@ -1929,7 +1929,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // urEventRelease requires access to the associated ur_queue_handle_t. // In urEventRelease, the reference counter of the Queue is decremented // to release it. - Queue->incrementRefCount(); + Queue->getRefCount().increment(); // SYCL RT does not track completion of the events, so it could // release a PI event as soon as that's not being waited in the app. @@ -1961,7 +1961,7 @@ void ur_queue_handle_t_::CaptureIndirectAccesses() { // SubmissionsCount turns to 0. We don't want to know how many times // allocation was retained by each submission. if (Pair.second) - Elem.second.incrementRefCount(); + Elem.second.getRefCount().increment(); } } Kernel->SubmissionsCount++; diff --git a/unified-runtime/source/adapters/level_zero/queue.hpp b/unified-runtime/source/adapters/level_zero/queue.hpp index 405929c8f0f0e..3e864fb3bb495 100644 --- a/unified-runtime/source/adapters/level_zero/queue.hpp +++ b/unified-runtime/source/adapters/level_zero/queue.hpp @@ -25,6 +25,7 @@ #include #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "device.hpp" extern "C" { @@ -692,6 +693,11 @@ struct ur_queue_handle_t_ : ur_object { // Pointer to the unified handle. ur_queue_handle_t_ *UnifiedHandle; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; // This helper function creates a ur_event_handle_t and associate a diff --git a/unified-runtime/source/adapters/level_zero/sampler.cpp b/unified-runtime/source/adapters/level_zero/sampler.cpp index 433043f11ae9a..738ce5fe9c951 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.cpp +++ b/unified-runtime/source/adapters/level_zero/sampler.cpp @@ -124,14 +124,14 @@ ur_result_t urSamplerCreate( ur_result_t urSamplerRetain( /// [in] handle of the sampler object to get access ur_sampler_handle_t Sampler) { - Sampler->incrementRefCount(); + Sampler->getRefCount().increment(); return UR_RESULT_SUCCESS; } ur_result_t urSamplerRelease( /// [in] handle of the sampler object to release ur_sampler_handle_t Sampler) { - if (!Sampler->decrementAndTest()) + if (!Sampler->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/sampler.hpp b/unified-runtime/source/adapters/level_zero/sampler.hpp index 9a834a05215d9..ba041795de674 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.hpp +++ b/unified-runtime/source/adapters/level_zero/sampler.hpp @@ -10,6 +10,7 @@ #pragma once #include "common.hpp" +#include "common/ur_ref_count.hpp" struct ur_sampler_handle_t_ : ur_object { ur_sampler_handle_t_(ze_sampler_handle_t Sampler) : ZeSampler{Sampler} {} @@ -18,6 +19,11 @@ struct ur_sampler_handle_t_ : ur_object { ze_sampler_handle_t ZeSampler; ZeStruct ZeSamplerDesc; + + URRefCount &getRefCount() noexcept { return RefCount; } + +private: + URRefCount RefCount; }; // Construct ZE sampler desc from UR sampler desc. diff --git a/unified-runtime/source/adapters/level_zero/usm.cpp b/unified-runtime/source/adapters/level_zero/usm.cpp index aa7a4c852f996..68b984fa16f8a 100644 --- a/unified-runtime/source/adapters/level_zero/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/usm.cpp @@ -523,14 +523,14 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t Pool) { - Pool->incrementRefCount(); + Pool->getRefCount().increment(); return UR_RESULT_SUCCESS; } ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t Pool) { - if (Pool->decrementAndTest()) { + if (Pool->getRefCount().decrementAndTest()) { std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); delete Pool; @@ -553,7 +553,7 @@ ur_result_t urUSMPoolGetInfo( switch (PropName) { case UR_USM_POOL_INFO_REFERENCE_COUNT: { - return ReturnValue(Pool->getRefCount()); + return ReturnValue(Pool->getRefCount().getCount()); } case UR_USM_POOL_INFO_CONTEXT: { return ReturnValue(Pool->Context); @@ -1250,7 +1250,7 @@ ur_result_t ZeMemFreeHelper(ur_context_handle_t Context, void *Ptr) { if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.decrementAndTest()) { + if (!It->second.getRefCount().decrementAndTest()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } @@ -1297,7 +1297,7 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.decrementAndTest()) { + if (!It->second.getRefCount().decrementAndTest()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/usm.hpp b/unified-runtime/source/adapters/level_zero/usm.hpp index b29ea29a7914d..e96fe993ef539 100644 --- a/unified-runtime/source/adapters/level_zero/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/usm.hpp @@ -9,13 +9,14 @@ //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" +#include +#include "common.hpp" +#include "common/ur_ref_count.hpp" #include "enqueued_pool.hpp" #include "event.hpp" #include "ur_api.h" #include "ur_pool_manager.hpp" -#include #include usm::DisjointPoolAllConfigs InitializeDisjointPoolConfig(); @@ -53,9 +54,13 @@ struct ur_usm_pool_handle_t_ : ur_object { ur_context_handle_t Context; + URRefCount &getRefCount() noexcept { return RefCount; } + private: UsmPool *getPool(const usm::pool_descriptor &Desc); usm::pool_manager PoolManager; + + URRefCount RefCount; }; // Exception type to pass allocation errors diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp index 2e5d74868cbaa..37b629bacaacc 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp @@ -258,7 +258,7 @@ urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - hCommandBuffer->incrementRefCount(); + hCommandBuffer->getRefCount().increment(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -266,7 +266,7 @@ urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - if (!hCommandBuffer->decrementAndTest()) + if (!hCommandBuffer->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; if (auto executionEvent = hCommandBuffer->getExecutionEventUnlocked()) { @@ -630,7 +630,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hCommandBuffer->getRefCount()}); + return ReturnValue(uint32_t{hCommandBuffer->getRefCount().getCount()}); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp index 109ec09ce24fd..c77943b09520f 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp @@ -9,15 +9,18 @@ //===----------------------------------------------------------------------===// #pragma once +#include + #include "../helpers/mutable_helpers.hpp" #include "command_list_manager.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "context.hpp" #include "kernel.hpp" #include "lockable.hpp" #include "queue_api.hpp" -#include #include + struct kernel_command_handle; struct ur_exp_command_buffer_handle_t_ : public ur_object { @@ -62,6 +65,8 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { ur_event_handle_t createEventIfRequested(ur_exp_command_buffer_sync_point_t *retSyncPoint); + URRefCount &getRefCount() noexcept { return RefCount; } + private: // Stores all sync points that are created by the command buffer. std::vector syncPoints; @@ -85,4 +90,6 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { ur_event_handle_t currentExecution = nullptr; v2::raii::cache_borrowed_event_pool eventPool; + + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index 326c1c9677fc3..46aeffb725219 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -916,7 +916,7 @@ ur_result_t ur_command_list_manager::appendNativeCommandExp( void ur_command_list_manager::recordSubmittedKernel( ur_kernel_handle_t hKernel) { submittedKernels.push_back(hKernel); - hKernel->incrementRefCount(); + hKernel->getRefCount().increment(); } ze_command_list_handle_t ur_command_list_manager::getZeCommandList() { diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index f5b37651f8542..c6bec8e4b7790 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -80,12 +80,12 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext, defaultUSMPool(this, nullptr), asyncPool(this, nullptr) {} ur_result_t ur_context_handle_t_::retain() { - incrementRefCount(); + RefCount.increment(); return UR_RESULT_SUCCESS; } ur_result_t ur_context_handle_t_::release() { - if (!decrementAndTest()) + if (!RefCount.decrementAndTest()) return UR_RESULT_SUCCESS; delete this; @@ -201,7 +201,7 @@ ur_result_t urContextGetInfo(ur_context_handle_t hContext, case UR_CONTEXT_INFO_NUM_DEVICES: return ReturnValue(uint32_t(hContext->getDevices().size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hContext->getRefCount()}); + return ReturnValue(uint32_t{hContext->getRefCount().getCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: // TODO: this is currently not implemented return ReturnValue(uint8_t{false}); diff --git a/unified-runtime/source/adapters/level_zero/v2/context.hpp b/unified-runtime/source/adapters/level_zero/v2/context.hpp index 8d3cf8ca05579..a9a41c59163fe 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.hpp @@ -14,6 +14,7 @@ #include "command_list_cache.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "event_pool_cache.hpp" #include "usm.hpp" @@ -64,6 +65,8 @@ struct ur_context_handle_t_ : ur_object { // For that the Device or its root devices need to be in the context. bool isValidDevice(ur_device_handle_t Device) const; + URRefCount &getRefCount() noexcept { return RefCount; } + private: const v2::raii::ze_context_handle_t hContext; const std::vector hDevices; @@ -81,4 +84,6 @@ struct ur_context_handle_t_ : ur_object { ur_usm_pool_handle_t_ defaultUSMPool; ur_usm_pool_handle_t_ asyncPool; std::list usmPoolHandles; + + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index bc520a09a0b66..70ce08a50b055 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -160,12 +160,12 @@ ze_event_handle_t ur_event_handle_t_::getZeEvent() const { } ur_result_t ur_event_handle_t_::retain() { - incrementRefCount(); + RefCount.increment(); return UR_RESULT_SUCCESS; } ur_result_t ur_event_handle_t_::release() { - if (!decrementAndTest()) + if (!RefCount.decrementAndTest()) return UR_RESULT_SUCCESS; if (event_pool) { @@ -258,7 +258,7 @@ ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, } } case UR_EVENT_INFO_REFERENCE_COUNT: { - return returnValue(hEvent->getRefCount()); + return returnValue(hEvent->getRefCount().getCount()); } case UR_EVENT_INFO_COMMAND_QUEUE: { auto urQueueHandle = reinterpret_cast(hEvent->getQueue()) - diff --git a/unified-runtime/source/adapters/level_zero/v2/event.hpp b/unified-runtime/source/adapters/level_zero/v2/event.hpp index 0e9386578a2f6..52a3068f095d8 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.hpp @@ -17,6 +17,7 @@ #include "adapters/level_zero/v2/queue_api.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "event_provider.hpp" namespace v2 { @@ -112,10 +113,14 @@ struct ur_event_handle_t_ : ur_object { uint64_t getEventStartTimestmap() const; uint64_t getEventEndTimestamp(); + URRefCount &getRefCount() noexcept { return RefCount; } + private: ur_event_handle_t_(ur_context_handle_t hContext, event_variant hZeEvent, v2::event_flags_t flags, v2::event_pool *pool); + URRefCount RefCount; + protected: ur_context_handle_t hContext; diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index a49f680999b84..4da798aaef490 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -52,8 +52,8 @@ void event_pool::free(ur_event_handle_t event) { freelist.push_back(event); // The event is still in the pool, so we need to increment the refcount - assert(event->getRefCount() == 0); - event->incrementRefCount(); + assert(event->getRefCount().getCount() == 0); + event->getRefCount().increment(); } event_provider *event_pool::getProvider() const { return provider.get(); } diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index 1ae6a7e8528a2..dff158e4372ab 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -97,7 +97,7 @@ ur_kernel_handle_t_::ur_kernel_handle_t_( } ur_result_t ur_kernel_handle_t_::release() { - if (!decrementAndTest()) + if (!RefCount.decrementAndTest()) return UR_RESULT_SUCCESS; // manually release kernels to allow errors to be propagated @@ -370,7 +370,7 @@ urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t hKernel) try { - hKernel->incrementRefCount(); + hKernel->getRefCount().increment(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -634,7 +634,7 @@ ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, spills.size()); } case UR_KERNEL_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hKernel->getRefCount()}); + return ReturnValue(uint32_t{hKernel->getRefCount().getCount()}); case UR_KERNEL_INFO_ATTRIBUTES: { auto attributes = hKernel->getSourceAttributes(); return ReturnValue(static_cast(attributes.data())); diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp index 0cabb888ac3be..e19b0ba08d329 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp @@ -13,6 +13,7 @@ #include "../program.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "memory.hpp" struct ur_single_device_kernel_t { @@ -91,6 +92,8 @@ struct ur_kernel_handle_t_ : ur_object { ze_command_list_handle_t cmdList, wait_list_view &waitListView); + URRefCount &getRefCount() noexcept { return RefCount; } + private: // Keep the program of the kernel. const ur_program_handle_t hProgram; @@ -116,4 +119,6 @@ struct ur_kernel_handle_t_ : ur_object { // pointer to any non-null kernel in deviceKernels ur_single_device_kernel_t *nonEmptyKernel; + + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index 49eba95fc25d2..644f05eb6ba04 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -671,7 +671,7 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, return returnValue(size_t{hMem->getBuffer()->getSize()}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return returnValue(hMem->getObject()->getRefCount()); + return returnValue(hMem->getRefCount().getCount()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -684,14 +684,14 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, } ur_result_t urMemRetain(ur_mem_handle_t hMem) try { - hMem->getObject()->incrementRefCount(); + hMem->getRefCount().increment(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urMemRelease(ur_mem_handle_t hMem) try { - if (!hMem->getObject()->decrementAndTest()) + if (!hMem->getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; delete hMem; diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.hpp b/unified-runtime/source/adapters/level_zero/v2/memory.hpp index 9c0dc66ef72b4..d0b18c9be182e 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.hpp @@ -19,6 +19,7 @@ #include "../image_common.hpp" #include "command_list_manager.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" using usm_unique_ptr_t = std::unique_ptr>; @@ -279,16 +280,10 @@ struct ur_mem_handle_t_ : ur::handle_base { mem); } - ur_object *getObject() { - return std::visit( - [](auto &&arg) -> ur_object * { - return static_cast(&arg); - }, - mem); - } - bool isImage() const { return std::holds_alternative(mem); } + URRefCount &getRefCount() noexcept { return RefCount; } + private: template ur_mem_handle_t_(std::in_place_type_t, Args &&...args) @@ -299,4 +294,7 @@ struct ur_mem_handle_t_ : ur::handle_base { ur_discrete_buffer_handle_t, ur_shared_buffer_handle_t, ur_mem_sub_buffer_t, ur_mem_image_t> mem; + +private: + URRefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp index 94dd024b2a667..d7c080d44607a 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp @@ -13,11 +13,12 @@ #pragma once +#include + #include "../common.hpp" #include "queue_immediate_in_order.hpp" #include "queue_immediate_out_of_order.hpp" #include -#include struct ur_queue_handle_t_ : ur::handle_base { using data_variant = std::variant { ur_result_t queueRetain() { return std::visit( [](auto &q) { - q.incrementRefCount(); + q.getRefCount().increment(); return UR_RESULT_SUCCESS; }, queue_data); @@ -59,7 +60,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRelease() { return std::visit( [queueHandle = this](auto &q) { - if (!q.decrementAndTest()) + if (!q.getRefCount().decrementAndTest()) return UR_RESULT_SUCCESS; delete queueHandle; return UR_RESULT_SUCCESS; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index d7f1b0ebf9404..b3cd35be52cc2 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -60,7 +60,7 @@ ur_queue_immediate_in_order_t::queueGetInfo(ur_queue_info_t propName, case UR_QUEUE_INFO_DEVICE: return ReturnValue(hDevice); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{getRefCount()}); + return ReturnValue(uint32_t{getRefCount().getCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(flags); case UR_QUEUE_INFO_SIZE: diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp index 362a6ea31c9f4..356609984a0f4 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp @@ -12,6 +12,7 @@ #include "../common.hpp" #include "../device.hpp" +#include "common/ur_ref_count.hpp" #include "context.hpp" #include "event.hpp" #include "event_pool_cache.hpp" @@ -32,6 +33,7 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { lockable commandListManager; ur_queue_flags_t flags; v2::raii::cache_borrowed_event_pool eventPool; + URRefCount RefCount; public: ur_queue_immediate_in_order_t(ur_context_handle_t, ur_device_handle_t, @@ -451,6 +453,8 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { numEventsInWaitList, phEventWaitList, createEventIfRequested(eventPool.get(), phEvent, this)); } + + URRefCount &getRefCount() noexcept { return RefCount; } }; } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index 84bac0db1b894..91205f9d7060f 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -332,7 +332,7 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { - hPool->incrementRefCount(); + hPool->getRefCount().increment(); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { return umf::umf2urResult(e); @@ -343,7 +343,7 @@ urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { - if (hPool->decrementAndTest()) { + if (hPool->getRefCount().decrementAndTest()) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; } @@ -369,7 +369,7 @@ ur_result_t urUSMPoolGetInfo( switch (propName) { case UR_USM_POOL_INFO_REFERENCE_COUNT: { - return ReturnValue(hPool->getRefCount()); + return ReturnValue(hPool->getRefCount().getCount()); } case UR_USM_POOL_INFO_CONTEXT: { return ReturnValue(hPool->getContextHandle()); diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.hpp b/unified-runtime/source/adapters/level_zero/v2/usm.hpp index ff33b5f6bbed1..e3a66930ad4cb 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.hpp @@ -14,6 +14,7 @@ #include "../enqueued_pool.hpp" #include "common.hpp" +#include "common/ur_ref_count.hpp" #include "event.hpp" #include "ur_pool_manager.hpp" @@ -49,9 +50,13 @@ struct ur_usm_pool_handle_t_ : ur_object { void cleanupPools(); void cleanupPoolsForQueue(void *hQueue); + URRefCount &getRefCount() noexcept { return RefCount; } + private: ur_context_handle_t hContext; usm::pool_manager poolManager; UsmPool *getPool(const usm::pool_descriptor &desc); + + URRefCount RefCount; }; diff --git a/unified-runtime/source/common/ur_ref_count.hpp b/unified-runtime/source/common/ur_ref_count.hpp new file mode 100644 index 0000000000000..4d1b5500477fc --- /dev/null +++ b/unified-runtime/source/common/ur_ref_count.hpp @@ -0,0 +1,29 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM + * Exceptions. See LICENSE.TXT + * + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ +#ifndef URREFCOUNT_HPP +#define URREFCOUNT_HPP 1 + +#include +#include + +class URRefCount { +public: + uint32_t getCount() const noexcept { return Count.load(); } + uint32_t increment() { return ++Count; } + uint32_t decrement() { return --Count; } + bool decrementAndTest() { return --Count == 0; } + void reset(uint32_t value = 1) { Count = value; } + +private: + std::atomic_uint32_t Count{1}; +}; + +#endif // URREFCOUNT_HPP diff --git a/unified-runtime/source/ur/ur.hpp b/unified-runtime/source/ur/ur.hpp index ec41529d9f575..5b0914868777a 100644 --- a/unified-runtime/source/ur/ur.hpp +++ b/unified-runtime/source/ur/ur.hpp @@ -228,15 +228,6 @@ template struct handle_base { // Handles are non-copyable. handle_base(const handle_base &) = delete; handle_base &operator=(const handle_base &) = delete; - - uint32_t getRefCount() const noexcept { return Count.load(); } - uint32_t incrementRefCount() { return ++Count; } - uint32_t decrementRefCount() { return --Count; } - bool decrementAndTest() { return --Count == 0; } - void resetRefCount(uint32_t value = 1) { Count = value; } - -private: - std::atomic_uint32_t Count{1}; }; template From 74db66f1ee69502e8cd40785027c836f368fa394 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Fri, 27 Jun 2025 12:46:54 +0100 Subject: [PATCH 5/7] Address feedback: rename URRefCount to RefCount and put inside ur namespace. Add a constructor for the class with a default value and remove use of reset() inside the Adapter constructor, using the RefCount constructor to set default value instead. --- .../source/adapters/level_zero/adapter.cpp | 4 +--- .../source/adapters/level_zero/adapter.hpp | 4 ++-- .../source/adapters/level_zero/command_buffer.hpp | 4 ++-- .../source/adapters/level_zero/common.hpp | 4 ++-- .../source/adapters/level_zero/context.hpp | 4 ++-- .../source/adapters/level_zero/device.hpp | 4 ++-- unified-runtime/source/adapters/level_zero/event.hpp | 4 ++-- .../source/adapters/level_zero/kernel.hpp | 4 ++-- .../source/adapters/level_zero/memory.hpp | 4 ++-- .../source/adapters/level_zero/physical_mem.hpp | 4 ++-- .../source/adapters/level_zero/program.hpp | 4 ++-- unified-runtime/source/adapters/level_zero/queue.hpp | 4 ++-- .../source/adapters/level_zero/sampler.hpp | 4 ++-- unified-runtime/source/adapters/level_zero/usm.hpp | 4 ++-- .../source/adapters/level_zero/v2/command_buffer.hpp | 4 ++-- .../source/adapters/level_zero/v2/context.hpp | 4 ++-- .../source/adapters/level_zero/v2/event.hpp | 4 ++-- .../source/adapters/level_zero/v2/kernel.hpp | 4 ++-- .../source/adapters/level_zero/v2/memory.hpp | 4 ++-- .../level_zero/v2/queue_immediate_in_order.hpp | 4 ++-- .../source/adapters/level_zero/v2/usm.hpp | 4 ++-- unified-runtime/source/common/ur_ref_count.hpp | 12 ++++++++++-- 22 files changed, 51 insertions(+), 45 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 899393553643b..eafaf8583ff87 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -296,13 +296,11 @@ Behavior Summary: SysMan initialization is skipped. */ ur_adapter_handle_t_::ur_adapter_handle_t_() - : handle_base(), logger(logger::get_logger("level_zero")) { + : handle_base(), logger(logger::get_logger("level_zero")), RefCount(0) { ZeInitDriversResult = ZE_RESULT_ERROR_UNINITIALIZED; ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; - RefCount.reset(0); - #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's // destruction of its context until after the UR Adapter is destroyed. diff --git a/unified-runtime/source/adapters/level_zero/adapter.hpp b/unified-runtime/source/adapters/level_zero/adapter.hpp index c75921c57130f..3549713c02068 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.hpp +++ b/unified-runtime/source/adapters/level_zero/adapter.hpp @@ -45,10 +45,10 @@ struct ur_adapter_handle_t_ : ur::handle_base { logger::Logger &logger; HMODULE processHandle = nullptr; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; extern ur_adapter_handle_t_ *GlobalAdapter; diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/command_buffer.hpp index 31f9bf5b114ab..bd9e8fd97310a 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.hpp @@ -151,8 +151,8 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { std::vector> CommandHandles; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index a94b83a891332..c33afe4144e75 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -259,10 +259,10 @@ struct MemAllocRecord : ur_object { // Zero runtime. ur_context_handle_t Context; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; extern usm::DisjointPoolAllConfigs DisjointPoolConfigInstance; diff --git a/unified-runtime/source/adapters/level_zero/context.hpp b/unified-runtime/source/adapters/level_zero/context.hpp index 70f9f1e3ae044..b1bb958f56257 100644 --- a/unified-runtime/source/adapters/level_zero/context.hpp +++ b/unified-runtime/source/adapters/level_zero/context.hpp @@ -359,7 +359,7 @@ struct ur_context_handle_t_ : ur_object { // Get handle to the L0 context ze_context_handle_t getZeHandle() const; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: enum EventFlags { @@ -408,7 +408,7 @@ struct ur_context_handle_t_ : ur_object { return &EventCaches[index]; } - URRefCount RefCount; + ur::RefCount RefCount; }; // Helper function to release the context, a caller must lock the platform-level diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index d52e240bc53b9..34dd5fcad725d 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -244,10 +244,10 @@ struct ur_device_handle_t_ : ur_object { // unique ephemeral identifer of the device in the adapter std::optional Id; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; inline std::vector diff --git a/unified-runtime/source/adapters/level_zero/event.hpp b/unified-runtime/source/adapters/level_zero/event.hpp index 51e187e29b72e..80565164902df 100644 --- a/unified-runtime/source/adapters/level_zero/event.hpp +++ b/unified-runtime/source/adapters/level_zero/event.hpp @@ -264,10 +264,10 @@ struct ur_event_handle_t_ : ur_object { // on async free to indicate when the allocation can be used again. ur_event_handle_t OriginAllocEvent = nullptr; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; // Helper function to implement zeHostSynchronize. diff --git a/unified-runtime/source/adapters/level_zero/kernel.hpp b/unified-runtime/source/adapters/level_zero/kernel.hpp index 0666edc7d8a27..b56ea0fa89cd9 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/kernel.hpp @@ -109,10 +109,10 @@ struct ur_kernel_handle_t_ : ur_object { ZeCache> ZeKernelProperties; ZeCache ZeKernelName; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; ur_result_t getZeKernel(ze_device_handle_t hDevice, ur_kernel_handle_t hKernel, diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index 280a7fc262b1d..d29a3ee791d50 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -91,7 +91,7 @@ struct ur_mem_handle_t_ : ur_object { // Method to get type of the derived object (image or buffer) bool isImage() const { return mem_type == mem_type_t::image; } - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } protected: ur_mem_handle_t_(mem_type_t type, ur_context_handle_t Context) @@ -106,7 +106,7 @@ struct ur_mem_handle_t_ : ur_object { ~ur_mem_handle_t_() {}; private: - URRefCount RefCount; + ur::RefCount RefCount; }; struct ur_buffer final : ur_mem_handle_t_ { diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/physical_mem.hpp index 235253cef04f8..8af78bfb3da43 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.hpp @@ -23,8 +23,8 @@ struct ur_physical_mem_handle_t_ : ur_object { // Keeps the PI context of this memory handle. ur_context_handle_t Context; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/program.hpp b/unified-runtime/source/adapters/level_zero/program.hpp index 29193afbe5ef0..cd73cd43c7031 100644 --- a/unified-runtime/source/adapters/level_zero/program.hpp +++ b/unified-runtime/source/adapters/level_zero/program.hpp @@ -227,7 +227,7 @@ struct ur_program_handle_t_ : ur_object { // UR_PROGRAM_INFO_BINARY_SIZES. const std::vector AssociatedDevices; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: struct DeviceData { @@ -268,5 +268,5 @@ struct ur_program_handle_t_ : ur_object { // TODO: Currently interoparability UR API does not support multiple devices. ze_module_handle_t InteropZeModule = nullptr; - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/queue.hpp b/unified-runtime/source/adapters/level_zero/queue.hpp index 3e864fb3bb495..daa33a76f47b0 100644 --- a/unified-runtime/source/adapters/level_zero/queue.hpp +++ b/unified-runtime/source/adapters/level_zero/queue.hpp @@ -694,10 +694,10 @@ struct ur_queue_handle_t_ : ur_object { // Pointer to the unified handle. ur_queue_handle_t_ *UnifiedHandle; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; // This helper function creates a ur_event_handle_t and associate a diff --git a/unified-runtime/source/adapters/level_zero/sampler.hpp b/unified-runtime/source/adapters/level_zero/sampler.hpp index ba041795de674..48584db343dda 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.hpp +++ b/unified-runtime/source/adapters/level_zero/sampler.hpp @@ -20,10 +20,10 @@ struct ur_sampler_handle_t_ : ur_object { ZeStruct ZeSamplerDesc; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: - URRefCount RefCount; + ur::RefCount RefCount; }; // Construct ZE sampler desc from UR sampler desc. diff --git a/unified-runtime/source/adapters/level_zero/usm.hpp b/unified-runtime/source/adapters/level_zero/usm.hpp index e96fe993ef539..489886b8a3bce 100644 --- a/unified-runtime/source/adapters/level_zero/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/usm.hpp @@ -54,13 +54,13 @@ struct ur_usm_pool_handle_t_ : ur_object { ur_context_handle_t Context; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: UsmPool *getPool(const usm::pool_descriptor &Desc); usm::pool_manager PoolManager; - URRefCount RefCount; + ur::RefCount RefCount; }; // Exception type to pass allocation errors diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp index c77943b09520f..2ff945048ff74 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp @@ -65,7 +65,7 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { ur_event_handle_t createEventIfRequested(ur_exp_command_buffer_sync_point_t *retSyncPoint); - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: // Stores all sync points that are created by the command buffer. @@ -91,5 +91,5 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { v2::raii::cache_borrowed_event_pool eventPool; - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/context.hpp b/unified-runtime/source/adapters/level_zero/v2/context.hpp index a9a41c59163fe..a060b7070f8be 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.hpp @@ -65,7 +65,7 @@ struct ur_context_handle_t_ : ur_object { // For that the Device or its root devices need to be in the context. bool isValidDevice(ur_device_handle_t Device) const; - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: const v2::raii::ze_context_handle_t hContext; @@ -85,5 +85,5 @@ struct ur_context_handle_t_ : ur_object { ur_usm_pool_handle_t_ asyncPool; std::list usmPoolHandles; - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/event.hpp b/unified-runtime/source/adapters/level_zero/v2/event.hpp index 52a3068f095d8..9320fa34d4e36 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.hpp @@ -113,13 +113,13 @@ struct ur_event_handle_t_ : ur_object { uint64_t getEventStartTimestmap() const; uint64_t getEventEndTimestamp(); - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: ur_event_handle_t_(ur_context_handle_t hContext, event_variant hZeEvent, v2::event_flags_t flags, v2::event_pool *pool); - URRefCount RefCount; + ur::RefCount RefCount; protected: ur_context_handle_t hContext; diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp index e19b0ba08d329..bafda2d1c963b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp @@ -92,7 +92,7 @@ struct ur_kernel_handle_t_ : ur_object { ze_command_list_handle_t cmdList, wait_list_view &waitListView); - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: // Keep the program of the kernel. @@ -120,5 +120,5 @@ struct ur_kernel_handle_t_ : ur_object { // pointer to any non-null kernel in deviceKernels ur_single_device_kernel_t *nonEmptyKernel; - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.hpp b/unified-runtime/source/adapters/level_zero/v2/memory.hpp index d0b18c9be182e..5ae8f810ed187 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.hpp @@ -282,7 +282,7 @@ struct ur_mem_handle_t_ : ur::handle_base { bool isImage() const { return std::holds_alternative(mem); } - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: template @@ -296,5 +296,5 @@ struct ur_mem_handle_t_ : ur::handle_base { mem; private: - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp index 356609984a0f4..7923e04923328 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp @@ -33,7 +33,7 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { lockable commandListManager; ur_queue_flags_t flags; v2::raii::cache_borrowed_event_pool eventPool; - URRefCount RefCount; + ur::RefCount RefCount; public: ur_queue_immediate_in_order_t(ur_context_handle_t, ur_device_handle_t, @@ -454,7 +454,7 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { createEventIfRequested(eventPool.get(), phEvent, this)); } - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } }; } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.hpp b/unified-runtime/source/adapters/level_zero/v2/usm.hpp index e3a66930ad4cb..4b38d71a0bf29 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.hpp @@ -50,7 +50,7 @@ struct ur_usm_pool_handle_t_ : ur_object { void cleanupPools(); void cleanupPoolsForQueue(void *hQueue); - URRefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount &getRefCount() noexcept { return RefCount; } private: ur_context_handle_t hContext; @@ -58,5 +58,5 @@ struct ur_usm_pool_handle_t_ : ur_object { UsmPool *getPool(const usm::pool_descriptor &desc); - URRefCount RefCount; + ur::RefCount RefCount; }; diff --git a/unified-runtime/source/common/ur_ref_count.hpp b/unified-runtime/source/common/ur_ref_count.hpp index 4d1b5500477fc..0545643b8dffb 100644 --- a/unified-runtime/source/common/ur_ref_count.hpp +++ b/unified-runtime/source/common/ur_ref_count.hpp @@ -14,8 +14,14 @@ #include #include -class URRefCount { +namespace ur { + +class RefCount { public: + RefCount(uint32_t count = 1) : Count(count) {} + RefCount(const RefCount &) = delete; + RefCount &operator=(const RefCount &) = delete; + uint32_t getCount() const noexcept { return Count.load(); } uint32_t increment() { return ++Count; } uint32_t decrement() { return --Count; } @@ -23,7 +29,9 @@ class URRefCount { void reset(uint32_t value = 1) { Count = value; } private: - std::atomic_uint32_t Count{1}; + std::atomic_uint32_t Count; }; +} // namespace ur + #endif // URREFCOUNT_HPP From a40a09b1aeda8790ac71a3a69dd4a40f439b9f39 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Fri, 27 Jun 2025 15:18:06 +0100 Subject: [PATCH 6/7] Address pr feedback: rename ref count functions increment -> retain and decrementAndTest -> release, and remove decrement as it doesn't get used. --- .../source/adapters/level_zero/adapter.cpp | 6 +++--- .../source/adapters/level_zero/async_alloc.cpp | 2 +- .../adapters/level_zero/command_buffer.cpp | 8 ++++---- .../source/adapters/level_zero/context.cpp | 4 ++-- .../source/adapters/level_zero/device.cpp | 4 ++-- .../source/adapters/level_zero/event.cpp | 12 ++++++------ .../source/adapters/level_zero/kernel.cpp | 4 ++-- .../source/adapters/level_zero/memory.cpp | 8 ++++---- .../source/adapters/level_zero/memory.hpp | 2 +- .../source/adapters/level_zero/physical_mem.cpp | 4 ++-- .../source/adapters/level_zero/program.cpp | 6 +++--- .../source/adapters/level_zero/queue.cpp | 16 ++++++++-------- .../source/adapters/level_zero/sampler.cpp | 4 ++-- .../source/adapters/level_zero/usm.cpp | 8 ++++---- .../adapters/level_zero/v2/command_buffer.cpp | 4 ++-- .../level_zero/v2/command_list_manager.cpp | 2 +- .../source/adapters/level_zero/v2/context.cpp | 4 ++-- .../source/adapters/level_zero/v2/event.cpp | 4 ++-- .../source/adapters/level_zero/v2/event_pool.cpp | 2 +- .../source/adapters/level_zero/v2/kernel.cpp | 4 ++-- .../source/adapters/level_zero/v2/memory.cpp | 4 ++-- .../adapters/level_zero/v2/queue_handle.hpp | 4 ++-- .../level_zero/v2/queue_immediate_in_order.cpp | 2 +- .../v2/queue_immediate_out_of_order.cpp | 2 +- .../v2/queue_immediate_out_of_order.hpp | 5 +++++ .../source/adapters/level_zero/v2/usm.cpp | 4 ++-- unified-runtime/source/common/ur_ref_count.hpp | 5 ++--- 27 files changed, 69 insertions(+), 65 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index eafaf8583ff87..19958d148035b 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -675,7 +675,7 @@ ur_result_t urAdapterGet( } *Adapters = GlobalAdapter; - if (GlobalAdapter->getRefCount().increment() == 0) { + if (GlobalAdapter->getRefCount().retain() == 0) { adapterStateInit(); } } @@ -692,7 +692,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { // NOTE: This does not require guarding with a mutex; the instant the ref // count hits zero, both Get and Retain are UB. - if (GlobalAdapter->getRefCount().decrementAndTest()) { + if (GlobalAdapter->getRefCount().release()) { auto result = adapterStateTeardown(); #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's @@ -711,7 +711,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { ur_result_t urAdapterRetain([[maybe_unused]] ur_adapter_handle_t Adapter) { assert(GlobalAdapter && GlobalAdapter == Adapter); - GlobalAdapter->getRefCount().increment(); + GlobalAdapter->getRefCount().retain(); return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/async_alloc.cpp b/unified-runtime/source/adapters/level_zero/async_alloc.cpp index d29bd6fd64ab6..201b5b4d17285 100644 --- a/unified-runtime/source/adapters/level_zero/async_alloc.cpp +++ b/unified-runtime/source/adapters/level_zero/async_alloc.cpp @@ -247,7 +247,7 @@ ur_result_t urEnqueueUSMFreeExp( } size_t size = umfPoolMallocUsableSize(hPool, Mem); - (*Event)->getRefCount().increment(); + (*Event)->getRefCount().retain(); usmPool->AsyncPool.insert(Mem, size, *Event, Queue); // Signal that USM free event was finished diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index 939779ef9cb44..b71d25b030294 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -840,13 +840,13 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) { - CommandBuffer->getRefCount().increment(); + CommandBuffer->getRefCount().retain(); return UR_RESULT_SUCCESS; } ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) { - if (!CommandBuffer->getRefCount().decrementAndTest()) + if (!CommandBuffer->getRefCount().release()) return UR_RESULT_SUCCESS; UR_CALL(waitForOngoingExecution(CommandBuffer)); @@ -1641,7 +1641,7 @@ ur_result_t enqueueImmediateAppendPath( if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->getRefCount().increment(); + (*Event)->getRefCount().retain(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(CommandListHelper, false, false)); @@ -1724,7 +1724,7 @@ ur_result_t enqueueWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer, if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->getRefCount().increment(); + (*Event)->getRefCount().retain(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(SignalCommandList, false /*IsBlocking*/, diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index a6f9450fefd9e..275f39c478115 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -61,7 +61,7 @@ ur_result_t urContextRetain( /// [in] handle of the context to get a reference of. ur_context_handle_t Context) { - Context->getRefCount().increment(); + Context->getRefCount().retain(); return UR_RESULT_SUCCESS; } @@ -251,7 +251,7 @@ ur_device_handle_t ur_context_handle_t_::getRootDevice() const { // from the list of tracked contexts. ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { - if (!Context->getRefCount().decrementAndTest()) + if (!Context->getRefCount().release()) return UR_RESULT_SUCCESS; if (IndirectAccessTrackingEnabled) { diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 6c43b28b80760..2cd11cb3a957e 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -1666,7 +1666,7 @@ ur_result_t urDeviceGetGlobalTimestamps( ur_result_t urDeviceRetain(ur_device_handle_t Device) { // The root-device ref-count remains unchanged (always 1). if (Device->isSubDevice()) { - Device->getRefCount().increment(); + Device->getRefCount().retain(); } return UR_RESULT_SUCCESS; } @@ -1674,7 +1674,7 @@ ur_result_t urDeviceRetain(ur_device_handle_t Device) { ur_result_t urDeviceRelease(ur_device_handle_t Device) { // Root devices are destroyed during the piTearDown process. if (Device->isSubDevice()) { - if (Device->getRefCount().decrementAndTest()) { + if (Device->getRefCount().release()) { delete Device; } } diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index b043f36a707f2..a31dc1ff72bf6 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -874,7 +874,7 @@ ur_result_t /// [in] handle of the event object urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { Event->RefCountExternal++; - Event->getRefCount().increment(); + Event->getRefCount().retain(); return UR_RESULT_SUCCESS; } @@ -1088,7 +1088,7 @@ ur_event_handle_t_::~ur_event_handle_t_() { ur_result_t urEventReleaseInternal(ur_event_handle_t Event, bool *isEventDeleted) { - if (!Event->getRefCount().decrementAndTest()) + if (!Event->getRefCount().release()) return UR_RESULT_SUCCESS; if (Event->OriginAllocEvent) { @@ -1524,7 +1524,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( std::shared_lock Lock(CurQueue->LastCommandEvent->Mutex); this->ZeEventList[0] = CurQueue->LastCommandEvent->ZeEvent; this->UrEventList[0] = CurQueue->LastCommandEvent; - this->UrEventList[0]->getRefCount().increment(); + this->UrEventList[0]->getRefCount().retain(); TmpListLength = 1; } else if (EventListLength > 0) { this->ZeEventList = new ze_event_handle_t[EventListLength]; @@ -1660,7 +1660,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( IsInternal, IsMultiDevice)); MultiDeviceZeEvent = MultiDeviceEvent->ZeEvent; const auto &ZeCommandList = CommandList->first; - EventList[I]->getRefCount().increment(); + EventList[I]->getRefCount().retain(); // Append a Barrier to wait on the original event while signalling the // new multi device event. @@ -1676,11 +1676,11 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( this->ZeEventList[TmpListLength] = MultiDeviceZeEvent; this->UrEventList[TmpListLength] = MultiDeviceEvent; - this->UrEventList[TmpListLength]->getRefCount().increment(); + this->UrEventList[TmpListLength]->getRefCount().retain(); } else { this->ZeEventList[TmpListLength] = EventList[I]->ZeEvent; this->UrEventList[TmpListLength] = EventList[I]; - this->UrEventList[TmpListLength]->getRefCount().increment(); + this->UrEventList[TmpListLength]->getRefCount().retain(); } if (QueueLock.has_value()) { diff --git a/unified-runtime/source/adapters/level_zero/kernel.cpp b/unified-runtime/source/adapters/level_zero/kernel.cpp index 29889beb13c4e..ab50dd6708a29 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/kernel.cpp @@ -938,7 +938,7 @@ ur_result_t urKernelGetSubGroupInfo( ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t Kernel) { - Kernel->getRefCount().increment(); + Kernel->getRefCount().retain(); return UR_RESULT_SUCCESS; } @@ -946,7 +946,7 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release ur_kernel_handle_t Kernel) { - if (!Kernel->getRefCount().decrementAndTest()) + if (!Kernel->getRefCount().release()) return UR_RESULT_SUCCESS; auto KernelProgram = Kernel->Program; diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index c855b42e856c3..e1139ff0e2c80 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -1052,7 +1052,7 @@ ur_result_t urEnqueueMemBufferMap( // Add the event to the command list. CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->getRefCount().increment(); + (*Event)->getRefCount().retain(); const auto &ZeCommandList = CommandList->first; const auto &WaitList = (*Event)->WaitList; @@ -1183,7 +1183,7 @@ ur_result_t urEnqueueMemUnmap( nullptr /*ForcedCmdQueue*/)); CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->getRefCount().increment(); + (*Event)->getRefCount().retain(); const auto &ZeCommandList = CommandList->first; @@ -1635,14 +1635,14 @@ ur_result_t urMemBufferCreate( ur_result_t urMemRetain( /// [in] handle of the memory object to get access ur_mem_handle_t Mem) { - Mem->getRefCount().increment(); + Mem->getRefCount().retain(); return UR_RESULT_SUCCESS; } ur_result_t urMemRelease( /// [in] handle of the memory object to release ur_mem_handle_t Mem) { - if (!Mem->getRefCount().decrementAndTest()) + if (!Mem->getRefCount().release()) return UR_RESULT_SUCCESS; if (Mem->isImage()) { diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index d29a3ee791d50..7bf7234293f3d 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -122,7 +122,7 @@ struct ur_buffer final : ur_mem_handle_t_ { : ur_mem_handle_t_(mem_type_t::buffer, Parent->UrContext), Size(Size), SubBuffer{{Parent, Origin}} { // Retain the Parent Buffer due to the Creation of the SubBuffer. - Parent->getRefCount().increment(); + Parent->getRefCount().retain(); } // Interop-buffer constructor diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index 480250a3af7ea..00debb98fe03e 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -42,12 +42,12 @@ ur_result_t urPhysicalMemCreate( } ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->getRefCount().increment(); + hPhysicalMem->getRefCount().retain(); return UR_RESULT_SUCCESS; } ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (!hPhysicalMem->getRefCount().decrementAndTest()) + if (!hPhysicalMem->getRefCount().release()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/program.cpp b/unified-runtime/source/adapters/level_zero/program.cpp index 921fa9fe991f0..edcff4e628729 100644 --- a/unified-runtime/source/adapters/level_zero/program.cpp +++ b/unified-runtime/source/adapters/level_zero/program.cpp @@ -558,14 +558,14 @@ ur_result_t urProgramLinkExp( ur_result_t urProgramRetain( /// [in] handle for the Program to retain ur_program_handle_t Program) { - Program->getRefCount().increment(); + Program->getRefCount().retain(); return UR_RESULT_SUCCESS; } ur_result_t urProgramRelease( /// [in] handle for the Program to release ur_program_handle_t Program) { - if (!Program->getRefCount().decrementAndTest()) + if (!Program->getRefCount().release()) return UR_RESULT_SUCCESS; delete Program; @@ -1115,7 +1115,7 @@ void ur_program_handle_t_::ur_release_program_resources(bool deletion) { // must be destroyed before the Module can be destroyed. So, be sure // to destroy build log before destroying the module. if (!deletion) { - if (!RefCount.decrementAndTest()) { + if (!RefCount.release()) { return; } } diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index b419dbc8f5443..cb01619041921 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -593,7 +593,7 @@ ur_result_t urQueueRetain( std::scoped_lock Lock(Queue->Mutex); Queue->RefCountExternal++; } - Queue->getRefCount().increment(); + Queue->getRefCount().retain(); return UR_RESULT_SUCCESS; } @@ -612,7 +612,7 @@ ur_result_t urQueueRelease( // internal reference count. When the External Reference count == 0, then // cleanup of the queue begins and the final decrement of the internal // reference count is completed. - static_cast(Queue->getRefCount().decrementAndTest()); + static_cast(Queue->getRefCount().release()); return UR_RESULT_SUCCESS; } @@ -1389,7 +1389,7 @@ ur_queue_handle_t_::executeCommandList(ur_command_list_ptr_t CommandList, if (!Event->HostVisibleEvent) { Event->HostVisibleEvent = reinterpret_cast(HostVisibleEvent); - HostVisibleEvent->getRefCount().increment(); + HostVisibleEvent->getRefCount().retain(); } } @@ -1550,7 +1550,7 @@ ur_result_t ur_queue_handle_t_::addEventToQueueCache(ur_event_handle_t Event) { } void ur_queue_handle_t_::active_barriers::add(ur_event_handle_t &Event) { - Event->getRefCount().increment(); + Event->getRefCount().retain(); Events.push_back(Event); } @@ -1588,7 +1588,7 @@ void ur_queue_handle_t_::clearEndTimeRecordings() { } ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) { - if (!Queue->getRefCount().decrementAndTest()) + if (!Queue->getRefCount().release()) return UR_RESULT_SUCCESS; for (auto &Cache : Queue->EventCaches) { @@ -1921,7 +1921,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // Append this Event to the CommandList, if any if (CommandList != Queue->CommandListMap.end()) { CommandList->second.append(*Event); - (*Event)->getRefCount().increment(); + (*Event)->getRefCount().retain(); } // We need to increment the reference counter here to avoid ur_queue_handle_t @@ -1929,7 +1929,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // urEventRelease requires access to the associated ur_queue_handle_t. // In urEventRelease, the reference counter of the Queue is decremented // to release it. - Queue->getRefCount().increment(); + Queue->getRefCount().retain(); // SYCL RT does not track completion of the events, so it could // release a PI event as soon as that's not being waited in the app. @@ -1961,7 +1961,7 @@ void ur_queue_handle_t_::CaptureIndirectAccesses() { // SubmissionsCount turns to 0. We don't want to know how many times // allocation was retained by each submission. if (Pair.second) - Elem.second.getRefCount().increment(); + Elem.second.getRefCount().retain(); } } Kernel->SubmissionsCount++; diff --git a/unified-runtime/source/adapters/level_zero/sampler.cpp b/unified-runtime/source/adapters/level_zero/sampler.cpp index 738ce5fe9c951..4d9d80b77c6e3 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.cpp +++ b/unified-runtime/source/adapters/level_zero/sampler.cpp @@ -124,14 +124,14 @@ ur_result_t urSamplerCreate( ur_result_t urSamplerRetain( /// [in] handle of the sampler object to get access ur_sampler_handle_t Sampler) { - Sampler->getRefCount().increment(); + Sampler->getRefCount().retain(); return UR_RESULT_SUCCESS; } ur_result_t urSamplerRelease( /// [in] handle of the sampler object to release ur_sampler_handle_t Sampler) { - if (!Sampler->getRefCount().decrementAndTest()) + if (!Sampler->getRefCount().release()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/usm.cpp b/unified-runtime/source/adapters/level_zero/usm.cpp index 68b984fa16f8a..1b7745924d476 100644 --- a/unified-runtime/source/adapters/level_zero/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/usm.cpp @@ -523,14 +523,14 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t Pool) { - Pool->getRefCount().increment(); + Pool->getRefCount().retain(); return UR_RESULT_SUCCESS; } ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t Pool) { - if (Pool->getRefCount().decrementAndTest()) { + if (Pool->getRefCount().release()) { std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); delete Pool; @@ -1250,7 +1250,7 @@ ur_result_t ZeMemFreeHelper(ur_context_handle_t Context, void *Ptr) { if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.getRefCount().decrementAndTest()) { + if (!It->second.getRefCount().release()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } @@ -1297,7 +1297,7 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.getRefCount().decrementAndTest()) { + if (!It->second.getRefCount().release()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp index 37b629bacaacc..2a7fe2a3e7be9 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp @@ -258,7 +258,7 @@ urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - hCommandBuffer->getRefCount().increment(); + hCommandBuffer->getRefCount().retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -266,7 +266,7 @@ urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - if (!hCommandBuffer->getRefCount().decrementAndTest()) + if (!hCommandBuffer->getRefCount().release()) return UR_RESULT_SUCCESS; if (auto executionEvent = hCommandBuffer->getExecutionEventUnlocked()) { diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index 46aeffb725219..78da24390735e 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -916,7 +916,7 @@ ur_result_t ur_command_list_manager::appendNativeCommandExp( void ur_command_list_manager::recordSubmittedKernel( ur_kernel_handle_t hKernel) { submittedKernels.push_back(hKernel); - hKernel->getRefCount().increment(); + hKernel->getRefCount().retain(); } ze_command_list_handle_t ur_command_list_manager::getZeCommandList() { diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index c6bec8e4b7790..3c64de388ea4d 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -80,12 +80,12 @@ ur_context_handle_t_::ur_context_handle_t_(ze_context_handle_t hContext, defaultUSMPool(this, nullptr), asyncPool(this, nullptr) {} ur_result_t ur_context_handle_t_::retain() { - RefCount.increment(); + RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t ur_context_handle_t_::release() { - if (!RefCount.decrementAndTest()) + if (!RefCount.release()) return UR_RESULT_SUCCESS; delete this; diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index 70ce08a50b055..3385d2f00b59c 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -160,12 +160,12 @@ ze_event_handle_t ur_event_handle_t_::getZeEvent() const { } ur_result_t ur_event_handle_t_::retain() { - RefCount.increment(); + RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t ur_event_handle_t_::release() { - if (!RefCount.decrementAndTest()) + if (!RefCount.release()) return UR_RESULT_SUCCESS; if (event_pool) { diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index 4da798aaef490..30c3c98809525 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -53,7 +53,7 @@ void event_pool::free(ur_event_handle_t event) { // The event is still in the pool, so we need to increment the refcount assert(event->getRefCount().getCount() == 0); - event->getRefCount().increment(); + event->getRefCount().retain(); } event_provider *event_pool::getProvider() const { return provider.get(); } diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index dff158e4372ab..9c4a9ae86a2cb 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -97,7 +97,7 @@ ur_kernel_handle_t_::ur_kernel_handle_t_( } ur_result_t ur_kernel_handle_t_::release() { - if (!RefCount.decrementAndTest()) + if (!RefCount.release()) return UR_RESULT_SUCCESS; // manually release kernels to allow errors to be propagated @@ -370,7 +370,7 @@ urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t hKernel) try { - hKernel->getRefCount().increment(); + hKernel->getRefCount().retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index 644f05eb6ba04..f235939ab82a6 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -684,14 +684,14 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, } ur_result_t urMemRetain(ur_mem_handle_t hMem) try { - hMem->getRefCount().increment(); + hMem->getRefCount().retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urMemRelease(ur_mem_handle_t hMem) try { - if (!hMem->getRefCount().decrementAndTest()) + if (!hMem->getRefCount().release()) return UR_RESULT_SUCCESS; delete hMem; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp index d7c080d44607a..cd5a8cc1fee86 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp @@ -51,7 +51,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRetain() { return std::visit( [](auto &q) { - q.getRefCount().increment(); + q.getRefCount().retain(); return UR_RESULT_SUCCESS; }, queue_data); @@ -60,7 +60,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRelease() { return std::visit( [queueHandle = this](auto &q) { - if (!q.getRefCount().decrementAndTest()) + if (!q.getRefCount().release()) return UR_RESULT_SUCCESS; delete queueHandle; return UR_RESULT_SUCCESS; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index b3cd35be52cc2..85e6c7e2503c9 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -60,7 +60,7 @@ ur_queue_immediate_in_order_t::queueGetInfo(ur_queue_info_t propName, case UR_QUEUE_INFO_DEVICE: return ReturnValue(hDevice); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{getRefCount().getCount()}); + return ReturnValue(uint32_t{RefCount.getCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(flags); case UR_QUEUE_INFO_SIZE: diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp index bfb6079af3ea5..f0344eba267f9 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.cpp @@ -54,7 +54,7 @@ ur_result_t ur_queue_immediate_out_of_order_t::queueGetInfo( case UR_QUEUE_INFO_DEVICE: return ReturnValue(hDevice); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{RefCount.load()}); + return ReturnValue(uint32_t{RefCount.getCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(flags); case UR_QUEUE_INFO_SIZE: diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index 1d0bf5636d58c..c34b208722630 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -11,6 +11,7 @@ #include "../common.hpp" #include "../device.hpp" +#include "common/ur_ref_count.hpp" #include "context.hpp" #include "event.hpp" @@ -49,6 +50,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object, ur_queue_t_ { numCommandLists; } + ur::RefCount RefCount; + public: ur_queue_immediate_out_of_order_t(ur_context_handle_t, ur_device_handle_t, uint32_t ordinal, @@ -503,6 +506,8 @@ struct ur_queue_immediate_out_of_order_t : ur_object, ur_queue_t_ { numEventsInWaitList, phEventWaitList, createEventIfRequested(eventPool.get(), phEvent, this)); } + + ur::RefCount &getRefCount() noexcept { return RefCount; } }; } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index 91205f9d7060f..85cd533b39704 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -332,7 +332,7 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { - hPool->getRefCount().increment(); + hPool->getRefCount().retain(); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { return umf::umf2urResult(e); @@ -343,7 +343,7 @@ urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { - if (hPool->getRefCount().decrementAndTest()) { + if (hPool->getRefCount().release()) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; } diff --git a/unified-runtime/source/common/ur_ref_count.hpp b/unified-runtime/source/common/ur_ref_count.hpp index 0545643b8dffb..7815e1dab65d0 100644 --- a/unified-runtime/source/common/ur_ref_count.hpp +++ b/unified-runtime/source/common/ur_ref_count.hpp @@ -23,9 +23,8 @@ class RefCount { RefCount &operator=(const RefCount &) = delete; uint32_t getCount() const noexcept { return Count.load(); } - uint32_t increment() { return ++Count; } - uint32_t decrement() { return --Count; } - bool decrementAndTest() { return --Count == 0; } + uint32_t retain() { return ++Count; } + bool release() { return --Count == 0; } void reset(uint32_t value = 1) { Count = value; } private: From 1c400e41519cfd86d914cbe76ee926c80cee64d5 Mon Sep 17 00:00:00 2001 From: Martin Morrison-Grant Date: Wed, 2 Jul 2025 09:37:17 +0100 Subject: [PATCH 7/7] Address pr feedback - make RefCount public and remove getRefCount(). --- .../source/adapters/level_zero/adapter.cpp | 8 ++++---- .../source/adapters/level_zero/adapter.hpp | 3 --- .../source/adapters/level_zero/async_alloc.cpp | 2 +- .../adapters/level_zero/command_buffer.cpp | 10 +++++----- .../adapters/level_zero/command_buffer.hpp | 3 --- .../source/adapters/level_zero/common.hpp | 3 --- .../source/adapters/level_zero/context.cpp | 6 +++--- .../source/adapters/level_zero/context.hpp | 4 +--- .../source/adapters/level_zero/device.cpp | 6 +++--- .../source/adapters/level_zero/device.hpp | 3 --- .../source/adapters/level_zero/event.cpp | 14 +++++++------- .../source/adapters/level_zero/event.hpp | 3 --- .../source/adapters/level_zero/kernel.cpp | 6 +++--- .../source/adapters/level_zero/kernel.hpp | 3 --- .../source/adapters/level_zero/memory.cpp | 10 +++++----- .../source/adapters/level_zero/memory.hpp | 7 ++----- .../adapters/level_zero/physical_mem.cpp | 6 +++--- .../adapters/level_zero/physical_mem.hpp | 3 --- .../source/adapters/level_zero/program.cpp | 6 +++--- .../source/adapters/level_zero/program.hpp | 4 +--- .../source/adapters/level_zero/queue.cpp | 18 +++++++++--------- .../source/adapters/level_zero/queue.hpp | 3 --- .../source/adapters/level_zero/sampler.cpp | 4 ++-- .../source/adapters/level_zero/sampler.hpp | 3 --- .../source/adapters/level_zero/usm.cpp | 10 +++++----- .../source/adapters/level_zero/usm.hpp | 4 +--- .../adapters/level_zero/v2/command_buffer.cpp | 6 +++--- .../adapters/level_zero/v2/command_buffer.hpp | 4 +--- .../level_zero/v2/command_list_manager.cpp | 2 +- .../source/adapters/level_zero/v2/context.cpp | 2 +- .../source/adapters/level_zero/v2/context.hpp | 4 +--- .../source/adapters/level_zero/v2/event.cpp | 2 +- .../source/adapters/level_zero/v2/event.hpp | 4 +--- .../adapters/level_zero/v2/event_pool.cpp | 4 ++-- .../source/adapters/level_zero/v2/kernel.cpp | 4 ++-- .../source/adapters/level_zero/v2/kernel.hpp | 4 +--- .../source/adapters/level_zero/v2/memory.cpp | 6 +++--- .../source/adapters/level_zero/v2/memory.hpp | 5 +---- .../adapters/level_zero/v2/queue_handle.hpp | 4 ++-- .../level_zero/v2/queue_immediate_in_order.hpp | 3 +-- .../v2/queue_immediate_out_of_order.hpp | 4 +--- .../source/adapters/level_zero/v2/usm.cpp | 6 +++--- .../source/adapters/level_zero/v2/usm.hpp | 4 +--- 43 files changed, 84 insertions(+), 136 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 19958d148035b..0c1abe7667bd3 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -675,7 +675,7 @@ ur_result_t urAdapterGet( } *Adapters = GlobalAdapter; - if (GlobalAdapter->getRefCount().retain() == 0) { + if (GlobalAdapter->RefCount.retain() == 0) { adapterStateInit(); } } @@ -692,7 +692,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { // NOTE: This does not require guarding with a mutex; the instant the ref // count hits zero, both Get and Retain are UB. - if (GlobalAdapter->getRefCount().release()) { + if (GlobalAdapter->RefCount.release()) { auto result = adapterStateTeardown(); #ifdef UR_STATIC_LEVEL_ZERO // Given static linking of the L0 Loader, we must delay the loader's @@ -711,7 +711,7 @@ ur_result_t urAdapterRelease([[maybe_unused]] ur_adapter_handle_t Adapter) { ur_result_t urAdapterRetain([[maybe_unused]] ur_adapter_handle_t Adapter) { assert(GlobalAdapter && GlobalAdapter == Adapter); - GlobalAdapter->getRefCount().retain(); + GlobalAdapter->RefCount.retain(); return UR_RESULT_SUCCESS; } @@ -740,7 +740,7 @@ ur_result_t urAdapterGetInfo(ur_adapter_handle_t, ur_adapter_info_t PropName, case UR_ADAPTER_INFO_BACKEND: return ReturnValue(UR_BACKEND_LEVEL_ZERO); case UR_ADAPTER_INFO_REFERENCE_COUNT: - return ReturnValue(GlobalAdapter->getRefCount().getCount()); + return ReturnValue(GlobalAdapter->RefCount.getCount()); case UR_ADAPTER_INFO_VERSION: { #ifdef UR_ADAPTER_LEVEL_ZERO_V2 uint32_t adapterVersion = 2; diff --git a/unified-runtime/source/adapters/level_zero/adapter.hpp b/unified-runtime/source/adapters/level_zero/adapter.hpp index 3549713c02068..890e39d296672 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.hpp +++ b/unified-runtime/source/adapters/level_zero/adapter.hpp @@ -45,9 +45,6 @@ struct ur_adapter_handle_t_ : ur::handle_base { logger::Logger &logger; HMODULE processHandle = nullptr; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/async_alloc.cpp b/unified-runtime/source/adapters/level_zero/async_alloc.cpp index 201b5b4d17285..6d55fda0f20b1 100644 --- a/unified-runtime/source/adapters/level_zero/async_alloc.cpp +++ b/unified-runtime/source/adapters/level_zero/async_alloc.cpp @@ -247,7 +247,7 @@ ur_result_t urEnqueueUSMFreeExp( } size_t size = umfPoolMallocUsableSize(hPool, Mem); - (*Event)->getRefCount().retain(); + (*Event)->RefCount.retain(); usmPool->AsyncPool.insert(Mem, size, *Event, Queue); // Signal that USM free event was finished diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/command_buffer.cpp index b71d25b030294..1e68069db51b2 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.cpp @@ -840,13 +840,13 @@ urCommandBufferCreateExp(ur_context_handle_t Context, ur_device_handle_t Device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t CommandBuffer) { - CommandBuffer->getRefCount().retain(); + CommandBuffer->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t CommandBuffer) { - if (!CommandBuffer->getRefCount().release()) + if (!CommandBuffer->RefCount.release()) return UR_RESULT_SUCCESS; UR_CALL(waitForOngoingExecution(CommandBuffer)); @@ -1641,7 +1641,7 @@ ur_result_t enqueueImmediateAppendPath( if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->getRefCount().retain(); + (*Event)->RefCount.retain(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(CommandListHelper, false, false)); @@ -1724,7 +1724,7 @@ ur_result_t enqueueWaitEventPath(ur_exp_command_buffer_handle_t CommandBuffer, if (CommandBuffer->CurrentSubmissionEvent) { UR_CALL(urEventReleaseInternal(CommandBuffer->CurrentSubmissionEvent)); } - (*Event)->getRefCount().retain(); + (*Event)->RefCount.retain(); CommandBuffer->CurrentSubmissionEvent = *Event; UR_CALL(Queue->executeCommandList(SignalCommandList, false /*IsBlocking*/, @@ -1848,7 +1848,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hCommandBuffer->getRefCount().getCount()}); + return ReturnValue(uint32_t{hCommandBuffer->RefCount.getCount()}); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; diff --git a/unified-runtime/source/adapters/level_zero/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/command_buffer.hpp index bd9e8fd97310a..9fe6ef83ab5bd 100644 --- a/unified-runtime/source/adapters/level_zero/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/command_buffer.hpp @@ -151,8 +151,5 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { std::vector> CommandHandles; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index c33afe4144e75..cfb19f4977004 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -259,9 +259,6 @@ struct MemAllocRecord : ur_object { // Zero runtime. ur_context_handle_t Context; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/context.cpp b/unified-runtime/source/adapters/level_zero/context.cpp index 275f39c478115..fe690f3673934 100644 --- a/unified-runtime/source/adapters/level_zero/context.cpp +++ b/unified-runtime/source/adapters/level_zero/context.cpp @@ -61,7 +61,7 @@ ur_result_t urContextRetain( /// [in] handle of the context to get a reference of. ur_context_handle_t Context) { - Context->getRefCount().retain(); + Context->RefCount.retain(); return UR_RESULT_SUCCESS; } @@ -113,7 +113,7 @@ ur_result_t urContextGetInfo( case UR_CONTEXT_INFO_NUM_DEVICES: return ReturnValue(uint32_t(Context->Devices.size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Context->getRefCount().getCount()}); + return ReturnValue(uint32_t{Context->RefCount.getCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: // 2D USM memcpy is supported. return ReturnValue(uint8_t{UseMemcpy2DOperations}); @@ -251,7 +251,7 @@ ur_device_handle_t ur_context_handle_t_::getRootDevice() const { // from the list of tracked contexts. ur_result_t ContextReleaseHelper(ur_context_handle_t Context) { - if (!Context->getRefCount().release()) + if (!Context->RefCount.release()) return UR_RESULT_SUCCESS; if (IndirectAccessTrackingEnabled) { diff --git a/unified-runtime/source/adapters/level_zero/context.hpp b/unified-runtime/source/adapters/level_zero/context.hpp index b1bb958f56257..fbcbceb71b7f0 100644 --- a/unified-runtime/source/adapters/level_zero/context.hpp +++ b/unified-runtime/source/adapters/level_zero/context.hpp @@ -359,7 +359,7 @@ struct ur_context_handle_t_ : ur_object { // Get handle to the L0 context ze_context_handle_t getZeHandle() const; - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: enum EventFlags { @@ -407,8 +407,6 @@ struct ur_context_handle_t_ : ur_object { return &EventCaches[index]; } - - ur::RefCount RefCount; }; // Helper function to release the context, a caller must lock the platform-level diff --git a/unified-runtime/source/adapters/level_zero/device.cpp b/unified-runtime/source/adapters/level_zero/device.cpp index 2cd11cb3a957e..7fdc25c7382f6 100644 --- a/unified-runtime/source/adapters/level_zero/device.cpp +++ b/unified-runtime/source/adapters/level_zero/device.cpp @@ -470,7 +470,7 @@ ur_result_t urDeviceGetInfo( return ReturnValue((uint32_t)Device->SubDevices.size()); } case UR_DEVICE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Device->getRefCount().getCount()}); + return ReturnValue(uint32_t{Device->RefCount.getCount()}); case UR_DEVICE_INFO_SUPPORTED_PARTITIONS: { // SYCL spec says: if this SYCL device cannot be partitioned into at least // two sub devices then the returned vector must be empty. @@ -1666,7 +1666,7 @@ ur_result_t urDeviceGetGlobalTimestamps( ur_result_t urDeviceRetain(ur_device_handle_t Device) { // The root-device ref-count remains unchanged (always 1). if (Device->isSubDevice()) { - Device->getRefCount().retain(); + Device->RefCount.retain(); } return UR_RESULT_SUCCESS; } @@ -1674,7 +1674,7 @@ ur_result_t urDeviceRetain(ur_device_handle_t Device) { ur_result_t urDeviceRelease(ur_device_handle_t Device) { // Root devices are destroyed during the piTearDown process. if (Device->isSubDevice()) { - if (Device->getRefCount().release()) { + if (Device->RefCount.release()) { delete Device; } } diff --git a/unified-runtime/source/adapters/level_zero/device.hpp b/unified-runtime/source/adapters/level_zero/device.hpp index 34dd5fcad725d..48d0d5c13c579 100644 --- a/unified-runtime/source/adapters/level_zero/device.hpp +++ b/unified-runtime/source/adapters/level_zero/device.hpp @@ -244,9 +244,6 @@ struct ur_device_handle_t_ : ur_object { // unique ephemeral identifer of the device in the adapter std::optional Id; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/event.cpp b/unified-runtime/source/adapters/level_zero/event.cpp index a31dc1ff72bf6..e1834376f0b41 100644 --- a/unified-runtime/source/adapters/level_zero/event.cpp +++ b/unified-runtime/source/adapters/level_zero/event.cpp @@ -505,7 +505,7 @@ ur_result_t urEventGetInfo( return ReturnValue(Result); } case UR_EVENT_INFO_REFERENCE_COUNT: { - return ReturnValue(Event->getRefCount().getCount()); + return ReturnValue(Event->RefCount.getCount()); } default: UR_LOG(ERR, "Unsupported ParamName in urEventGetInfo: ParamName={}(0x{})", @@ -874,7 +874,7 @@ ur_result_t /// [in] handle of the event object urEventRetain(/** [in] handle of the event object */ ur_event_handle_t Event) { Event->RefCountExternal++; - Event->getRefCount().retain(); + Event->RefCount.retain(); return UR_RESULT_SUCCESS; } @@ -1088,7 +1088,7 @@ ur_event_handle_t_::~ur_event_handle_t_() { ur_result_t urEventReleaseInternal(ur_event_handle_t Event, bool *isEventDeleted) { - if (!Event->getRefCount().release()) + if (!Event->RefCount.release()) return UR_RESULT_SUCCESS; if (Event->OriginAllocEvent) { @@ -1524,7 +1524,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( std::shared_lock Lock(CurQueue->LastCommandEvent->Mutex); this->ZeEventList[0] = CurQueue->LastCommandEvent->ZeEvent; this->UrEventList[0] = CurQueue->LastCommandEvent; - this->UrEventList[0]->getRefCount().retain(); + this->UrEventList[0]->RefCount.retain(); TmpListLength = 1; } else if (EventListLength > 0) { this->ZeEventList = new ze_event_handle_t[EventListLength]; @@ -1660,7 +1660,7 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( IsInternal, IsMultiDevice)); MultiDeviceZeEvent = MultiDeviceEvent->ZeEvent; const auto &ZeCommandList = CommandList->first; - EventList[I]->getRefCount().retain(); + EventList[I]->RefCount.retain(); // Append a Barrier to wait on the original event while signalling the // new multi device event. @@ -1676,11 +1676,11 @@ ur_result_t ur_ze_event_list_t::createAndRetainUrZeEventList( this->ZeEventList[TmpListLength] = MultiDeviceZeEvent; this->UrEventList[TmpListLength] = MultiDeviceEvent; - this->UrEventList[TmpListLength]->getRefCount().retain(); + this->UrEventList[TmpListLength]->RefCount.retain(); } else { this->ZeEventList[TmpListLength] = EventList[I]->ZeEvent; this->UrEventList[TmpListLength] = EventList[I]; - this->UrEventList[TmpListLength]->getRefCount().retain(); + this->UrEventList[TmpListLength]->RefCount.retain(); } if (QueueLock.has_value()) { diff --git a/unified-runtime/source/adapters/level_zero/event.hpp b/unified-runtime/source/adapters/level_zero/event.hpp index 80565164902df..c89ee5097c8e8 100644 --- a/unified-runtime/source/adapters/level_zero/event.hpp +++ b/unified-runtime/source/adapters/level_zero/event.hpp @@ -264,9 +264,6 @@ struct ur_event_handle_t_ : ur_object { // on async free to indicate when the allocation can be used again. ur_event_handle_t OriginAllocEvent = nullptr; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/kernel.cpp b/unified-runtime/source/adapters/level_zero/kernel.cpp index ab50dd6708a29..838cb96dc59ed 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/kernel.cpp @@ -787,7 +787,7 @@ ur_result_t urKernelGetInfo( } } case UR_KERNEL_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Kernel->getRefCount().getCount()}); + return ReturnValue(uint32_t{Kernel->RefCount.getCount()}); case UR_KERNEL_INFO_ATTRIBUTES: try { uint32_t Size; @@ -938,7 +938,7 @@ ur_result_t urKernelGetSubGroupInfo( ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t Kernel) { - Kernel->getRefCount().retain(); + Kernel->RefCount.retain(); return UR_RESULT_SUCCESS; } @@ -946,7 +946,7 @@ ur_result_t urKernelRetain( ur_result_t urKernelRelease( /// [in] handle for the Kernel to release ur_kernel_handle_t Kernel) { - if (!Kernel->getRefCount().release()) + if (!Kernel->RefCount.release()) return UR_RESULT_SUCCESS; auto KernelProgram = Kernel->Program; diff --git a/unified-runtime/source/adapters/level_zero/kernel.hpp b/unified-runtime/source/adapters/level_zero/kernel.hpp index b56ea0fa89cd9..131dba270c05d 100644 --- a/unified-runtime/source/adapters/level_zero/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/kernel.hpp @@ -109,9 +109,6 @@ struct ur_kernel_handle_t_ : ur_object { ZeCache> ZeKernelProperties; ZeCache ZeKernelName; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index e1139ff0e2c80..3b1158645e77a 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -1052,7 +1052,7 @@ ur_result_t urEnqueueMemBufferMap( // Add the event to the command list. CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->getRefCount().retain(); + (*Event)->RefCount.retain(); const auto &ZeCommandList = CommandList->first; const auto &WaitList = (*Event)->WaitList; @@ -1183,7 +1183,7 @@ ur_result_t urEnqueueMemUnmap( nullptr /*ForcedCmdQueue*/)); CommandList->second.append(reinterpret_cast(*Event)); - (*Event)->getRefCount().retain(); + (*Event)->RefCount.retain(); const auto &ZeCommandList = CommandList->first; @@ -1635,14 +1635,14 @@ ur_result_t urMemBufferCreate( ur_result_t urMemRetain( /// [in] handle of the memory object to get access ur_mem_handle_t Mem) { - Mem->getRefCount().retain(); + Mem->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urMemRelease( /// [in] handle of the memory object to release ur_mem_handle_t Mem) { - if (!Mem->getRefCount().release()) + if (!Mem->RefCount.release()) return UR_RESULT_SUCCESS; if (Mem->isImage()) { @@ -1848,7 +1848,7 @@ ur_result_t urMemGetInfo( return ReturnValue(size_t{Buffer->Size}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(Buffer->getRefCount().getCount()); + return ReturnValue(Buffer->RefCount.getCount()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index 7bf7234293f3d..f58f189b21c77 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -91,7 +91,7 @@ struct ur_mem_handle_t_ : ur_object { // Method to get type of the derived object (image or buffer) bool isImage() const { return mem_type == mem_type_t::image; } - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; protected: ur_mem_handle_t_(mem_type_t type, ur_context_handle_t Context) @@ -104,9 +104,6 @@ struct ur_mem_handle_t_ : ur_object { // Since the destructor isn't virtual, callers must destruct it via ur_buffer // or ur_image ~ur_mem_handle_t_() {}; - -private: - ur::RefCount RefCount; }; struct ur_buffer final : ur_mem_handle_t_ { @@ -122,7 +119,7 @@ struct ur_buffer final : ur_mem_handle_t_ { : ur_mem_handle_t_(mem_type_t::buffer, Parent->UrContext), Size(Size), SubBuffer{{Parent, Origin}} { // Retain the Parent Buffer due to the Creation of the SubBuffer. - Parent->getRefCount().retain(); + Parent->RefCount.retain(); } // Interop-buffer constructor diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.cpp b/unified-runtime/source/adapters/level_zero/physical_mem.cpp index 00debb98fe03e..8786dc2f784d1 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.cpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.cpp @@ -42,12 +42,12 @@ ur_result_t urPhysicalMemCreate( } ur_result_t urPhysicalMemRetain(ur_physical_mem_handle_t hPhysicalMem) { - hPhysicalMem->getRefCount().retain(); + hPhysicalMem->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urPhysicalMemRelease(ur_physical_mem_handle_t hPhysicalMem) { - if (!hPhysicalMem->getRefCount().release()) + if (!hPhysicalMem->RefCount.release()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { @@ -68,7 +68,7 @@ ur_result_t urPhysicalMemGetInfo(ur_physical_mem_handle_t hPhysicalMem, switch (propName) { case UR_PHYSICAL_MEM_INFO_REFERENCE_COUNT: { - return ReturnValue(hPhysicalMem->getRefCount().getCount()); + return ReturnValue(hPhysicalMem->RefCount.getCount()); } default: return UR_RESULT_ERROR_UNSUPPORTED_ENUMERATION; diff --git a/unified-runtime/source/adapters/level_zero/physical_mem.hpp b/unified-runtime/source/adapters/level_zero/physical_mem.hpp index 8af78bfb3da43..8e0ab261bcfd4 100644 --- a/unified-runtime/source/adapters/level_zero/physical_mem.hpp +++ b/unified-runtime/source/adapters/level_zero/physical_mem.hpp @@ -23,8 +23,5 @@ struct ur_physical_mem_handle_t_ : ur_object { // Keeps the PI context of this memory handle. ur_context_handle_t Context; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/program.cpp b/unified-runtime/source/adapters/level_zero/program.cpp index edcff4e628729..f41f9f6faf9ff 100644 --- a/unified-runtime/source/adapters/level_zero/program.cpp +++ b/unified-runtime/source/adapters/level_zero/program.cpp @@ -558,14 +558,14 @@ ur_result_t urProgramLinkExp( ur_result_t urProgramRetain( /// [in] handle for the Program to retain ur_program_handle_t Program) { - Program->getRefCount().retain(); + Program->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urProgramRelease( /// [in] handle for the Program to release ur_program_handle_t Program) { - if (!Program->getRefCount().release()) + if (!Program->RefCount.release()) return UR_RESULT_SUCCESS; delete Program; @@ -708,7 +708,7 @@ ur_result_t urProgramGetInfo( switch (PropName) { case UR_PROGRAM_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Program->getRefCount().getCount()}); + return ReturnValue(uint32_t{Program->RefCount.getCount()}); case UR_PROGRAM_INFO_CONTEXT: return ReturnValue(Program->Context); case UR_PROGRAM_INFO_NUM_DEVICES: diff --git a/unified-runtime/source/adapters/level_zero/program.hpp b/unified-runtime/source/adapters/level_zero/program.hpp index cd73cd43c7031..fb2cc1f12ee5e 100644 --- a/unified-runtime/source/adapters/level_zero/program.hpp +++ b/unified-runtime/source/adapters/level_zero/program.hpp @@ -227,7 +227,7 @@ struct ur_program_handle_t_ : ur_object { // UR_PROGRAM_INFO_BINARY_SIZES. const std::vector AssociatedDevices; - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: struct DeviceData { @@ -267,6 +267,4 @@ struct ur_program_handle_t_ : ur_object { // handle from the program. // TODO: Currently interoparability UR API does not support multiple devices. ze_module_handle_t InteropZeModule = nullptr; - - ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/queue.cpp b/unified-runtime/source/adapters/level_zero/queue.cpp index cb01619041921..4cb06f1348b7f 100644 --- a/unified-runtime/source/adapters/level_zero/queue.cpp +++ b/unified-runtime/source/adapters/level_zero/queue.cpp @@ -369,7 +369,7 @@ ur_result_t urQueueGetInfo( case UR_QUEUE_INFO_DEVICE: return ReturnValue(Queue->Device); case UR_QUEUE_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{Queue->getRefCount().getCount()}); + return ReturnValue(uint32_t{Queue->RefCount.getCount()}); case UR_QUEUE_INFO_FLAGS: return ReturnValue(Queue->Properties); case UR_QUEUE_INFO_SIZE: @@ -593,7 +593,7 @@ ur_result_t urQueueRetain( std::scoped_lock Lock(Queue->Mutex); Queue->RefCountExternal++; } - Queue->getRefCount().retain(); + Queue->RefCount.retain(); return UR_RESULT_SUCCESS; } @@ -612,7 +612,7 @@ ur_result_t urQueueRelease( // internal reference count. When the External Reference count == 0, then // cleanup of the queue begins and the final decrement of the internal // reference count is completed. - static_cast(Queue->getRefCount().release()); + static_cast(Queue->RefCount.release()); return UR_RESULT_SUCCESS; } @@ -1389,7 +1389,7 @@ ur_queue_handle_t_::executeCommandList(ur_command_list_ptr_t CommandList, if (!Event->HostVisibleEvent) { Event->HostVisibleEvent = reinterpret_cast(HostVisibleEvent); - HostVisibleEvent->getRefCount().retain(); + HostVisibleEvent->RefCount.retain(); } } @@ -1550,7 +1550,7 @@ ur_result_t ur_queue_handle_t_::addEventToQueueCache(ur_event_handle_t Event) { } void ur_queue_handle_t_::active_barriers::add(ur_event_handle_t &Event) { - Event->getRefCount().retain(); + Event->RefCount.retain(); Events.push_back(Event); } @@ -1588,7 +1588,7 @@ void ur_queue_handle_t_::clearEndTimeRecordings() { } ur_result_t urQueueReleaseInternal(ur_queue_handle_t Queue) { - if (!Queue->getRefCount().release()) + if (!Queue->RefCount.release()) return UR_RESULT_SUCCESS; for (auto &Cache : Queue->EventCaches) { @@ -1921,7 +1921,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // Append this Event to the CommandList, if any if (CommandList != Queue->CommandListMap.end()) { CommandList->second.append(*Event); - (*Event)->getRefCount().retain(); + (*Event)->RefCount.retain(); } // We need to increment the reference counter here to avoid ur_queue_handle_t @@ -1929,7 +1929,7 @@ ur_result_t createEventAndAssociateQueue(ur_queue_handle_t Queue, // urEventRelease requires access to the associated ur_queue_handle_t. // In urEventRelease, the reference counter of the Queue is decremented // to release it. - Queue->getRefCount().retain(); + Queue->RefCount.retain(); // SYCL RT does not track completion of the events, so it could // release a PI event as soon as that's not being waited in the app. @@ -1961,7 +1961,7 @@ void ur_queue_handle_t_::CaptureIndirectAccesses() { // SubmissionsCount turns to 0. We don't want to know how many times // allocation was retained by each submission. if (Pair.second) - Elem.second.getRefCount().retain(); + Elem.second.RefCount.retain(); } } Kernel->SubmissionsCount++; diff --git a/unified-runtime/source/adapters/level_zero/queue.hpp b/unified-runtime/source/adapters/level_zero/queue.hpp index daa33a76f47b0..a81ef2fecd56a 100644 --- a/unified-runtime/source/adapters/level_zero/queue.hpp +++ b/unified-runtime/source/adapters/level_zero/queue.hpp @@ -694,9 +694,6 @@ struct ur_queue_handle_t_ : ur_object { // Pointer to the unified handle. ur_queue_handle_t_ *UnifiedHandle; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/sampler.cpp b/unified-runtime/source/adapters/level_zero/sampler.cpp index 4d9d80b77c6e3..e048cb5fe4408 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.cpp +++ b/unified-runtime/source/adapters/level_zero/sampler.cpp @@ -124,14 +124,14 @@ ur_result_t urSamplerCreate( ur_result_t urSamplerRetain( /// [in] handle of the sampler object to get access ur_sampler_handle_t Sampler) { - Sampler->getRefCount().retain(); + Sampler->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t urSamplerRelease( /// [in] handle of the sampler object to release ur_sampler_handle_t Sampler) { - if (!Sampler->getRefCount().release()) + if (!Sampler->RefCount.release()) return UR_RESULT_SUCCESS; if (checkL0LoaderTeardown()) { diff --git a/unified-runtime/source/adapters/level_zero/sampler.hpp b/unified-runtime/source/adapters/level_zero/sampler.hpp index 48584db343dda..29b2b9617c822 100644 --- a/unified-runtime/source/adapters/level_zero/sampler.hpp +++ b/unified-runtime/source/adapters/level_zero/sampler.hpp @@ -20,9 +20,6 @@ struct ur_sampler_handle_t_ : ur_object { ZeStruct ZeSamplerDesc; - ur::RefCount &getRefCount() noexcept { return RefCount; } - -private: ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/usm.cpp b/unified-runtime/source/adapters/level_zero/usm.cpp index 1b7745924d476..e8dfb89df145c 100644 --- a/unified-runtime/source/adapters/level_zero/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/usm.cpp @@ -523,14 +523,14 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t Pool) { - Pool->getRefCount().retain(); + Pool->RefCount.retain(); return UR_RESULT_SUCCESS; } ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t Pool) { - if (Pool->getRefCount().release()) { + if (Pool->RefCount.release()) { std::scoped_lock ContextLock(Pool->Context->Mutex); Pool->Context->UsmPoolHandles.remove(Pool); delete Pool; @@ -553,7 +553,7 @@ ur_result_t urUSMPoolGetInfo( switch (PropName) { case UR_USM_POOL_INFO_REFERENCE_COUNT: { - return ReturnValue(Pool->getRefCount().getCount()); + return ReturnValue(Pool->RefCount.getCount()); } case UR_USM_POOL_INFO_CONTEXT: { return ReturnValue(Pool->Context); @@ -1250,7 +1250,7 @@ ur_result_t ZeMemFreeHelper(ur_context_handle_t Context, void *Ptr) { if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.getRefCount().release()) { + if (!It->second.RefCount.release()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } @@ -1297,7 +1297,7 @@ ur_result_t USMFreeHelper(ur_context_handle_t Context, void *Ptr, if (It == std::end(Context->MemAllocs)) { die("All memory allocations must be tracked!"); } - if (!It->second.getRefCount().release()) { + if (!It->second.RefCount.release()) { // Memory can't be deallocated yet. return UR_RESULT_SUCCESS; } diff --git a/unified-runtime/source/adapters/level_zero/usm.hpp b/unified-runtime/source/adapters/level_zero/usm.hpp index 489886b8a3bce..f99a2f79c87b2 100644 --- a/unified-runtime/source/adapters/level_zero/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/usm.hpp @@ -54,13 +54,11 @@ struct ur_usm_pool_handle_t_ : ur_object { ur_context_handle_t Context; - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: UsmPool *getPool(const usm::pool_descriptor &Desc); usm::pool_manager PoolManager; - - ur::RefCount RefCount; }; // Exception type to pass allocation errors diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp index 2a7fe2a3e7be9..bbc200bac6bab 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.cpp @@ -258,7 +258,7 @@ urCommandBufferCreateExp(ur_context_handle_t context, ur_device_handle_t device, ur_result_t urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - hCommandBuffer->getRefCount().retain(); + hCommandBuffer->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -266,7 +266,7 @@ urCommandBufferRetainExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { ur_result_t urCommandBufferReleaseExp(ur_exp_command_buffer_handle_t hCommandBuffer) try { - if (!hCommandBuffer->getRefCount().release()) + if (!hCommandBuffer->RefCount.release()) return UR_RESULT_SUCCESS; if (auto executionEvent = hCommandBuffer->getExecutionEventUnlocked()) { @@ -630,7 +630,7 @@ urCommandBufferGetInfoExp(ur_exp_command_buffer_handle_t hCommandBuffer, switch (propName) { case UR_EXP_COMMAND_BUFFER_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hCommandBuffer->getRefCount().getCount()}); + return ReturnValue(uint32_t{hCommandBuffer->RefCount.getCount()}); case UR_EXP_COMMAND_BUFFER_INFO_DESCRIPTOR: { ur_exp_command_buffer_desc_t Descriptor{}; Descriptor.stype = UR_STRUCTURE_TYPE_EXP_COMMAND_BUFFER_DESC; diff --git a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp index 2ff945048ff74..c7f1d585a5b77 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_buffer.hpp @@ -65,7 +65,7 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { ur_event_handle_t createEventIfRequested(ur_exp_command_buffer_sync_point_t *retSyncPoint); - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: // Stores all sync points that are created by the command buffer. @@ -90,6 +90,4 @@ struct ur_exp_command_buffer_handle_t_ : public ur_object { ur_event_handle_t currentExecution = nullptr; v2::raii::cache_borrowed_event_pool eventPool; - - ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp index 78da24390735e..4532e6ded3e1b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/command_list_manager.cpp @@ -916,7 +916,7 @@ ur_result_t ur_command_list_manager::appendNativeCommandExp( void ur_command_list_manager::recordSubmittedKernel( ur_kernel_handle_t hKernel) { submittedKernels.push_back(hKernel); - hKernel->getRefCount().retain(); + hKernel->RefCount.retain(); } ze_command_list_handle_t ur_command_list_manager::getZeCommandList() { diff --git a/unified-runtime/source/adapters/level_zero/v2/context.cpp b/unified-runtime/source/adapters/level_zero/v2/context.cpp index 3c64de388ea4d..bf054a493f656 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.cpp @@ -201,7 +201,7 @@ ur_result_t urContextGetInfo(ur_context_handle_t hContext, case UR_CONTEXT_INFO_NUM_DEVICES: return ReturnValue(uint32_t(hContext->getDevices().size())); case UR_CONTEXT_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hContext->getRefCount().getCount()}); + return ReturnValue(uint32_t{hContext->RefCount.getCount()}); case UR_CONTEXT_INFO_USM_MEMCPY2D_SUPPORT: // TODO: this is currently not implemented return ReturnValue(uint8_t{false}); diff --git a/unified-runtime/source/adapters/level_zero/v2/context.hpp b/unified-runtime/source/adapters/level_zero/v2/context.hpp index a060b7070f8be..b1500092a727b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/context.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/context.hpp @@ -65,7 +65,7 @@ struct ur_context_handle_t_ : ur_object { // For that the Device or its root devices need to be in the context. bool isValidDevice(ur_device_handle_t Device) const; - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: const v2::raii::ze_context_handle_t hContext; @@ -84,6 +84,4 @@ struct ur_context_handle_t_ : ur_object { ur_usm_pool_handle_t_ defaultUSMPool; ur_usm_pool_handle_t_ asyncPool; std::list usmPoolHandles; - - ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/event.cpp b/unified-runtime/source/adapters/level_zero/v2/event.cpp index 3385d2f00b59c..2c3c4b9a8685c 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.cpp @@ -258,7 +258,7 @@ ur_result_t urEventGetInfo(ur_event_handle_t hEvent, ur_event_info_t propName, } } case UR_EVENT_INFO_REFERENCE_COUNT: { - return returnValue(hEvent->getRefCount().getCount()); + return returnValue(hEvent->RefCount.getCount()); } case UR_EVENT_INFO_COMMAND_QUEUE: { auto urQueueHandle = reinterpret_cast(hEvent->getQueue()) - diff --git a/unified-runtime/source/adapters/level_zero/v2/event.hpp b/unified-runtime/source/adapters/level_zero/v2/event.hpp index 9320fa34d4e36..9a31c47358947 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/event.hpp @@ -113,14 +113,12 @@ struct ur_event_handle_t_ : ur_object { uint64_t getEventStartTimestmap() const; uint64_t getEventEndTimestamp(); - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: ur_event_handle_t_(ur_context_handle_t hContext, event_variant hZeEvent, v2::event_flags_t flags, v2::event_pool *pool); - ur::RefCount RefCount; - protected: ur_context_handle_t hContext; diff --git a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp index 30c3c98809525..f5d6d3429dbdf 100644 --- a/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/event_pool.cpp @@ -52,8 +52,8 @@ void event_pool::free(ur_event_handle_t event) { freelist.push_back(event); // The event is still in the pool, so we need to increment the refcount - assert(event->getRefCount().getCount() == 0); - event->getRefCount().retain(); + assert(event->RefCount.getCount() == 0); + event->RefCount.retain(); } event_provider *event_pool::getProvider() const { return provider.get(); } diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp index 9c4a9ae86a2cb..173b51ffc42a5 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.cpp @@ -370,7 +370,7 @@ urKernelCreateWithNativeHandle(ur_native_handle_t hNativeKernel, ur_result_t urKernelRetain( /// [in] handle for the Kernel to retain ur_kernel_handle_t hKernel) try { - hKernel->getRefCount().retain(); + hKernel->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); @@ -634,7 +634,7 @@ ur_result_t urKernelGetInfo(ur_kernel_handle_t hKernel, spills.size()); } case UR_KERNEL_INFO_REFERENCE_COUNT: - return ReturnValue(uint32_t{hKernel->getRefCount().getCount()}); + return ReturnValue(uint32_t{hKernel->RefCount.getCount()}); case UR_KERNEL_INFO_ATTRIBUTES: { auto attributes = hKernel->getSourceAttributes(); return ReturnValue(static_cast(attributes.data())); diff --git a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp index bafda2d1c963b..9c823760c42f2 100644 --- a/unified-runtime/source/adapters/level_zero/v2/kernel.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/kernel.hpp @@ -92,7 +92,7 @@ struct ur_kernel_handle_t_ : ur_object { ze_command_list_handle_t cmdList, wait_list_view &waitListView); - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: // Keep the program of the kernel. @@ -119,6 +119,4 @@ struct ur_kernel_handle_t_ : ur_object { // pointer to any non-null kernel in deviceKernels ur_single_device_kernel_t *nonEmptyKernel; - - ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index f235939ab82a6..9c39a97d163e8 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -671,7 +671,7 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, return returnValue(size_t{hMem->getBuffer()->getSize()}); } case UR_MEM_INFO_REFERENCE_COUNT: { - return returnValue(hMem->getRefCount().getCount()); + return returnValue(hMem->RefCount.getCount()); } default: { return UR_RESULT_ERROR_INVALID_ENUMERATION; @@ -684,14 +684,14 @@ ur_result_t urMemGetInfo(ur_mem_handle_t hMem, ur_mem_info_t propName, } ur_result_t urMemRetain(ur_mem_handle_t hMem) try { - hMem->getRefCount().retain(); + hMem->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (...) { return exceptionToResult(std::current_exception()); } ur_result_t urMemRelease(ur_mem_handle_t hMem) try { - if (!hMem->getRefCount().release()) + if (!hMem->RefCount.release()) return UR_RESULT_SUCCESS; delete hMem; diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.hpp b/unified-runtime/source/adapters/level_zero/v2/memory.hpp index 5ae8f810ed187..7201df57c9509 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.hpp @@ -282,7 +282,7 @@ struct ur_mem_handle_t_ : ur::handle_base { bool isImage() const { return std::holds_alternative(mem); } - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: template @@ -294,7 +294,4 @@ struct ur_mem_handle_t_ : ur::handle_base { ur_discrete_buffer_handle_t, ur_shared_buffer_handle_t, ur_mem_sub_buffer_t, ur_mem_image_t> mem; - -private: - ur::RefCount RefCount; }; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp index cd5a8cc1fee86..c414f79a46d71 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_handle.hpp @@ -51,7 +51,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRetain() { return std::visit( [](auto &q) { - q.getRefCount().retain(); + q.RefCount.retain(); return UR_RESULT_SUCCESS; }, queue_data); @@ -60,7 +60,7 @@ struct ur_queue_handle_t_ : ur::handle_base { ur_result_t queueRelease() { return std::visit( [queueHandle = this](auto &q) { - if (!q.getRefCount().release()) + if (!q.RefCount.release()) return UR_RESULT_SUCCESS; delete queueHandle; return UR_RESULT_SUCCESS; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp index 7923e04923328..74b37d1b40eb3 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.hpp @@ -33,7 +33,6 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { lockable commandListManager; ur_queue_flags_t flags; v2::raii::cache_borrowed_event_pool eventPool; - ur::RefCount RefCount; public: ur_queue_immediate_in_order_t(ur_context_handle_t, ur_device_handle_t, @@ -454,7 +453,7 @@ struct ur_queue_immediate_in_order_t : ur_object, ur_queue_t_ { createEventIfRequested(eventPool.get(), phEvent, this)); } - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; }; } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp index c34b208722630..07e8743154ded 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_out_of_order.hpp @@ -50,8 +50,6 @@ struct ur_queue_immediate_out_of_order_t : ur_object, ur_queue_t_ { numCommandLists; } - ur::RefCount RefCount; - public: ur_queue_immediate_out_of_order_t(ur_context_handle_t, ur_device_handle_t, uint32_t ordinal, @@ -507,7 +505,7 @@ struct ur_queue_immediate_out_of_order_t : ur_object, ur_queue_t_ { createEventIfRequested(eventPool.get(), phEvent, this)); } - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; }; } // namespace v2 diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.cpp b/unified-runtime/source/adapters/level_zero/v2/usm.cpp index 85cd533b39704..f95d86e1c33e9 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.cpp @@ -332,7 +332,7 @@ ur_result_t urUSMPoolCreate( ur_result_t /// [in] pointer to USM memory pool urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { - hPool->getRefCount().retain(); + hPool->RefCount.retain(); return UR_RESULT_SUCCESS; } catch (umf_result_t e) { return umf::umf2urResult(e); @@ -343,7 +343,7 @@ urUSMPoolRetain(ur_usm_pool_handle_t hPool) try { ur_result_t /// [in] pointer to USM memory pool urUSMPoolRelease(ur_usm_pool_handle_t hPool) try { - if (hPool->getRefCount().release()) { + if (hPool->RefCount.release()) { hPool->getContextHandle()->removeUsmPool(hPool); delete hPool; } @@ -369,7 +369,7 @@ ur_result_t urUSMPoolGetInfo( switch (propName) { case UR_USM_POOL_INFO_REFERENCE_COUNT: { - return ReturnValue(hPool->getRefCount().getCount()); + return ReturnValue(hPool->RefCount.getCount()); } case UR_USM_POOL_INFO_CONTEXT: { return ReturnValue(hPool->getContextHandle()); diff --git a/unified-runtime/source/adapters/level_zero/v2/usm.hpp b/unified-runtime/source/adapters/level_zero/v2/usm.hpp index 4b38d71a0bf29..35e3446b82abc 100644 --- a/unified-runtime/source/adapters/level_zero/v2/usm.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/usm.hpp @@ -50,13 +50,11 @@ struct ur_usm_pool_handle_t_ : ur_object { void cleanupPools(); void cleanupPoolsForQueue(void *hQueue); - ur::RefCount &getRefCount() noexcept { return RefCount; } + ur::RefCount RefCount; private: ur_context_handle_t hContext; usm::pool_manager poolManager; UsmPool *getPool(const usm::pool_descriptor &desc); - - ur::RefCount RefCount; };