From ddf35afb204f7e193b35679cb7f62b56148e44e8 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Mon, 2 Oct 2023 11:17:31 -0500 Subject: [PATCH 1/2] [libc] Change the GPU to use builtin memory functions Summary: The GPU build is special in the sense that we always know that up-to-date `clang` is always going to be the compiler. This allows us to rely directly on builtins, which allow us to push a lot of this complexity into the backend. Backend implementations are favored on the GPU because it allows us to do a lot more target specific optimizations. This patch changes over the common memory functions to use builtin versions when building for AMDGPU or NVPTX. --- .../src/string/memory_utils/generic/builtin.h | 47 +++++++++++++++++++ libc/src/string/memory_utils/inline_bcmp.h | 6 ++- libc/src/string/memory_utils/inline_memcpy.h | 5 +- libc/src/string/memory_utils/inline_memmove.h | 5 +- libc/src/string/memory_utils/inline_memset.h | 5 +- 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 libc/src/string/memory_utils/generic/builtin.h diff --git a/libc/src/string/memory_utils/generic/builtin.h b/libc/src/string/memory_utils/generic/builtin.h new file mode 100644 index 0000000000000..4ac8b3b4a305d --- /dev/null +++ b/libc/src/string/memory_utils/generic/builtin.h @@ -0,0 +1,47 @@ +//===-- Trivial builtin implementations ----------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_BUILTIN_H +#define LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_BUILTIN_H + +#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/config.h" // LIBC_HAS_BUILTIN +#include "src/string/memory_utils/utils.h" // Ptr, CPtr + +#include // size_t + +namespace LIBC_NAMESPACE { + +static_assert(LIBC_HAS_BUILTIN(__builtin_memcpy), "Builtin not defined"); +static_assert(LIBC_HAS_BUILTIN(__builtin_memset), "Builtin not defined"); +static_assert(LIBC_HAS_BUILTIN(__builtin_memmove), "Builtin not defined"); +static_assert(LIBC_HAS_BUILTIN(__builtin_bcmp), "Builtin not defined"); + +[[maybe_unused]] LIBC_INLINE void +inline_memcpy_builtin(Ptr dst, CPtr src, size_t count, size_t offset = 0) { + __builtin_memcpy(dst + offset, src + offset, count); +} + +[[maybe_unused]] LIBC_INLINE void inline_memmove_builtin(Ptr dst, CPtr src, + size_t count) { + __builtin_memmove(dst, src, count); +} + +[[maybe_unused]] LIBC_INLINE static void +inline_memset_builtin(Ptr dst, uint8_t value, size_t count, size_t offset = 0) { + __builtin_memset(dst + offset, value, count); +} + +[[maybe_unused]] LIBC_INLINE int +inline_bcmp_builtin(CPtr p1, CPtr p2, size_t count, size_t offset = 0) { + return __builtin_bcmp(p1 + offset, p2 + offset, count); +} + +} // namespace LIBC_NAMESPACE + +#endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_BUILTIN_H diff --git a/libc/src/string/memory_utils/inline_bcmp.h b/libc/src/string/memory_utils/inline_bcmp.h index b1c981d859e02..e859c2ea2e5e4 100644 --- a/libc/src/string/memory_utils/inline_bcmp.h +++ b/libc/src/string/memory_utils/inline_bcmp.h @@ -23,9 +23,13 @@ #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_bcmp.h" #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) +// FIXME: The NVPTX builtin for `bcmp` currently does not work. +#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_NVPTX) #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_byte_per_byte +#elif defined(LIBC_TARGET_ARCH_IS_GPU) +#include "src/string/memory_utils/generic/builtin.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_builtin #else #error "Unsupported architecture" #endif diff --git a/libc/src/string/memory_utils/inline_memcpy.h b/libc/src/string/memory_utils/inline_memcpy.h index 0b8a7848da87b..a92bf4ddf881d 100644 --- a/libc/src/string/memory_utils/inline_memcpy.h +++ b/libc/src/string/memory_utils/inline_memcpy.h @@ -28,9 +28,12 @@ #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_memcpy.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) +#elif defined(LIBC_TARGET_ARCH_IS_ARM) #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_byte_per_byte +#elif defined(LIBC_TARGET_ARCH_IS_GPU) +#include "src/string/memory_utils/generic/builtin.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMCPY inline_memcpy_builtin #else #error "Unsupported architecture" #endif diff --git a/libc/src/string/memory_utils/inline_memmove.h b/libc/src/string/memory_utils/inline_memmove.h index 0d31e10eaff28..f72ea24ab538d 100644 --- a/libc/src/string/memory_utils/inline_memmove.h +++ b/libc/src/string/memory_utils/inline_memmove.h @@ -20,9 +20,12 @@ #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_memmove.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) +#elif defined(LIBC_TARGET_ARCH_IS_ARM) #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_byte_per_byte +#elif defined(LIBC_TARGET_ARCH_IS_GPU) +#include "src/string/memory_utils/generic/builtin.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMMOVE inline_memmove_builtin #else #error "Unsupported architecture" #endif diff --git a/libc/src/string/memory_utils/inline_memset.h b/libc/src/string/memory_utils/inline_memset.h index f20ae45fa753b..1c07c1ca4bffc 100644 --- a/libc/src/string/memory_utils/inline_memset.h +++ b/libc/src/string/memory_utils/inline_memset.h @@ -24,9 +24,12 @@ #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_memset.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_riscv -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) +#elif defined(LIBC_TARGET_ARCH_IS_ARM) #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_byte_per_byte +#elif defined(LIBC_TARGET_ARCH_IS_GPU) +#include "src/string/memory_utils/generic/builtin.h" +#define LIBC_SRC_STRING_MEMORY_UTILS_MEMSET inline_memset_builtin #else #error "Unsupported architecture" #endif From a94ee146f8bbe3457dfc5bb09f80e9726a30f7d5 Mon Sep 17 00:00:00 2001 From: Joseph Huber Date: Tue, 3 Oct 2023 10:52:14 -0500 Subject: [PATCH 2/2] Remove memcmp and bcmp changes as they are not supported by the backend --- libc/src/string/memory_utils/generic/builtin.h | 6 ------ libc/src/string/memory_utils/inline_bcmp.h | 6 +----- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/libc/src/string/memory_utils/generic/builtin.h b/libc/src/string/memory_utils/generic/builtin.h index 4ac8b3b4a305d..5239329f653b3 100644 --- a/libc/src/string/memory_utils/generic/builtin.h +++ b/libc/src/string/memory_utils/generic/builtin.h @@ -20,7 +20,6 @@ namespace LIBC_NAMESPACE { static_assert(LIBC_HAS_BUILTIN(__builtin_memcpy), "Builtin not defined"); static_assert(LIBC_HAS_BUILTIN(__builtin_memset), "Builtin not defined"); static_assert(LIBC_HAS_BUILTIN(__builtin_memmove), "Builtin not defined"); -static_assert(LIBC_HAS_BUILTIN(__builtin_bcmp), "Builtin not defined"); [[maybe_unused]] LIBC_INLINE void inline_memcpy_builtin(Ptr dst, CPtr src, size_t count, size_t offset = 0) { @@ -37,11 +36,6 @@ inline_memset_builtin(Ptr dst, uint8_t value, size_t count, size_t offset = 0) { __builtin_memset(dst + offset, value, count); } -[[maybe_unused]] LIBC_INLINE int -inline_bcmp_builtin(CPtr p1, CPtr p2, size_t count, size_t offset = 0) { - return __builtin_bcmp(p1 + offset, p2 + offset, count); -} - } // namespace LIBC_NAMESPACE #endif // LLVM_LIBC_SRC_STRING_MEMORY_UTILS_GENERIC_BUILTIN_H diff --git a/libc/src/string/memory_utils/inline_bcmp.h b/libc/src/string/memory_utils/inline_bcmp.h index e859c2ea2e5e4..b1c981d859e02 100644 --- a/libc/src/string/memory_utils/inline_bcmp.h +++ b/libc/src/string/memory_utils/inline_bcmp.h @@ -23,13 +23,9 @@ #elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV) #include "src/string/memory_utils/riscv/inline_bcmp.h" #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_riscv -// FIXME: The NVPTX builtin for `bcmp` currently does not work. -#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_NVPTX) +#elif defined(LIBC_TARGET_ARCH_IS_ARM) || defined(LIBC_TARGET_ARCH_IS_GPU) #include "src/string/memory_utils/generic/byte_per_byte.h" #define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_byte_per_byte -#elif defined(LIBC_TARGET_ARCH_IS_GPU) -#include "src/string/memory_utils/generic/builtin.h" -#define LIBC_SRC_STRING_MEMORY_UTILS_BCMP inline_bcmp_builtin #else #error "Unsupported architecture" #endif