AArch64: Add libcall impl declarations for __arm_sc* memory functions

[arsenm]

These were bypassing the ordinary libcall emission mechanism. Make sure
we have entries in RuntimeLibcalls, which should include all possible
calls the compiler could emit.

Fixes not emitting the # prefix in the arm64ec case.

[arsenm]

• ARM: Start moving runtime libcalls into tablegen #146084
• ARM: Unconditionally set eabi libcall calling convs in RuntimeLibcalls #146083
• RuntimeLibcalls: Remove table of soft float compare cond codes #146082
• MSP430: Move libcall CC setting to RuntimeLibcallsInfo #146081
• Lanai: Use TableGen to set libcall calling conventions #146080
• TableGen: Handle setting runtime libcall calling conventions #144980
• RuntimeLibcalls: Associate calling convention with libcall impls #144979
• TableGen: Allow defining sets of runtime libraries #144978
• AArch64: Add libcall impl declarations for __arm_sc* memory functions #144977 👈 (View in Graphite)
• XCore: Declare libcalls used for align 4 memcpy #144976
• Hexagon: Add libcall declarations for special memcpy #144975
• ARM: Add runtime libcall definitions for aebi memory functions #144974
• TableGen: Generate enum for runtime libcall implementations #144973
• TableGen: Add runtime libcall backend #144972
• PowerPC: Stop reporting memcpy as an alias of memmove on AIX #143836
• main

This stack of pull requests is managed by Graphite. Learn more about stacking.

[llvmbot]

@llvm/pr-subscribers-backend-aarch64

@llvm/pr-subscribers-llvm-ir

Author: Matt Arsenault (arsenm)

Changes

These were bypassing the ordinary libcall emission mechanism. Make sure
we have entries in RuntimeLibcalls, which should include all possible
calls the compiler could emit.

Fixes not emitting the # prefix in the arm64ec case.

---

Full diff: https://github.com/llvm/llvm-project/pull/144977.diff

4 Files Affected:

• (modified) llvm/include/llvm/IR/RuntimeLibcalls.td (+9)
• (modified) llvm/lib/IR/RuntimeLibcalls.cpp (+10-1)
• (modified) llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp (+7-8)
• (modified) llvm/test/CodeGen/AArch64/arm64ec-builtins.ll (+3-6)

diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td
index 57ad6f09e8b57..1d9f02dcf8ba8 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.td
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.td
@@ -357,6 +357,11 @@ multiclass LibmLongDoubleLibCall<string libcall_basename = !toupper(NAME),
                            !strconcat(rtbasename, "l")>;
 }
 
+// AArch64 calls
+def SC_MEMCPY : RuntimeLibcall;
+def SC_MEMMOVE : RuntimeLibcall;
+def SC_MEMSET : RuntimeLibcall;
+
 // ARM EABI calls
 def AEABI_MEMCPY4 : RuntimeLibcall; // Align 4
 def AEABI_MEMCPY8 : RuntimeLibcall; // Align 8
@@ -985,6 +990,10 @@ defset list<RuntimeLibcallImpl> AArch64LibcallImpls = {
     defm __aarch64_ldeor#MemSize
         : AtomicOrderSizeLibcallImpl<"OUTLINE_ATOMIC_LDEOR"#MemSize>;
   }
+
+  def __arm_sc_memcpy : RuntimeLibcallImpl<SC_MEMCPY>;
+  def __arm_sc_memmove : RuntimeLibcallImpl<SC_MEMMOVE>;
+  def __arm_sc_memset : RuntimeLibcallImpl<SC_MEMSET>;
 }
 
 foreach libcall = AArch64LibcallImpls in {
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 0f92371f05529..67f872b4574b1 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -521,8 +521,17 @@ void RuntimeLibcallsInfo::initLibcalls(const Triple &TT,
   }
 
   if (TT.isAArch64()) {
-    if (TT.isWindowsArm64EC())
+    if (TT.isWindowsArm64EC()) {
       setWindowsArm64LibCallNameOverrides();
+      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::arm64ec___arm_sc_memcpy);
+      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::arm64ec___arm_sc_memmove);
+      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::arm64ec___arm_sc_memset);
+    } else {
+      setLibcallImpl(RTLIB::SC_MEMCPY, RTLIB::__arm_sc_memcpy);
+      setLibcallImpl(RTLIB::SC_MEMMOVE, RTLIB::__arm_sc_memmove);
+      setLibcallImpl(RTLIB::SC_MEMSET, RTLIB::__arm_sc_memset);
+    }
+
     setAArch64LibcallNames(*this, TT);
   } else if (TT.isARM() || TT.isThumb()) {
     setARMLibcallNames(*this, TT, FloatABI, EABIVersion);
diff --git a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
index 90f6fc2ea664b..d719f234b27f7 100644
--- a/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64SelectionDAGInfo.cpp
@@ -164,35 +164,34 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
   const AArch64Subtarget &STI =
       DAG.getMachineFunction().getSubtarget<AArch64Subtarget>();
   const AArch64TargetLowering *TLI = STI.getTargetLowering();
-  SDValue Symbol;
   TargetLowering::ArgListEntry DstEntry;
   DstEntry.Ty = PointerType::getUnqual(*DAG.getContext());
   DstEntry.Node = Dst;
   TargetLowering::ArgListTy Args;
   Args.push_back(DstEntry);
-  EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
 
+  RTLIB::Libcall NewLC;
   switch (LC) {
   case RTLIB::MEMCPY: {
+    NewLC = RTLIB::SC_MEMCPY;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memcpy", PointerVT);
     Entry.Node = Src;
     Args.push_back(Entry);
     break;
   }
   case RTLIB::MEMMOVE: {
+    NewLC = RTLIB::SC_MEMMOVE;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = PointerType::getUnqual(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memmove", PointerVT);
     Entry.Node = Src;
     Args.push_back(Entry);
     break;
   }
   case RTLIB::MEMSET: {
+    NewLC = RTLIB::SC_MEMSET;
     TargetLowering::ArgListEntry Entry;
     Entry.Ty = Type::getInt32Ty(*DAG.getContext());
-    Symbol = DAG.getExternalSymbol("__arm_sc_memset", PointerVT);
     Src = DAG.getZExtOrTrunc(Src, DL, MVT::i32);
     Entry.Node = Src;
     Args.push_back(Entry);
@@ -202,17 +201,17 @@ SDValue AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(
     return SDValue();
   }
 
+  EVT PointerVT = TLI->getPointerTy(DAG.getDataLayout());
+  SDValue Symbol = DAG.getExternalSymbol(TLI->getLibcallName(NewLC), PointerVT);
   TargetLowering::ArgListEntry SizeEntry;
   SizeEntry.Node = Size;
   SizeEntry.Ty = DAG.getDataLayout().getIntPtrType(*DAG.getContext());
   Args.push_back(SizeEntry);
-  assert(Symbol->getOpcode() == ISD::ExternalSymbol &&
-         "Function name is not set");
 
   TargetLowering::CallLoweringInfo CLI(DAG);
   PointerType *RetTy = PointerType::getUnqual(*DAG.getContext());
   CLI.setDebugLoc(DL).setChain(Chain).setLibCallee(
-      TLI->getLibcallCallingConv(LC), RetTy, Symbol, std::move(Args));
+      TLI->getLibcallCallingConv(NewLC), RetTy, Symbol, std::move(Args));
   return TLI->LowerCallTo(CLI).second;
 }
 
diff --git a/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll b/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
index 38416310b3536..911b6fa8eff4c 100644
--- a/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
+++ b/llvm/test/CodeGen/AArch64/arm64ec-builtins.ll
@@ -46,27 +46,24 @@ define float @f6(float %val, i32 %a) {
 @dst = global [512 x i8] zeroinitializer, align 1
 @src = global [512 x i8] zeroinitializer, align 1
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memcpy(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memcpy":
-; CHECK: bl __arm_sc_memcpy
+; CHECK: bl "#__arm_sc_memcpy"
 
   tail call void @llvm.memcpy.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
   ret void
 }
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memmove(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memmove":
-; CHECK: bl __arm_sc_memmove
+; CHECK: bl "#__arm_sc_memmove"
   tail call void @llvm.memmove.p0.p0.i64(ptr align 1 @dst, ptr nonnull align 1 @src, i64 %n, i1 false)
   ret void
 }
 
-; FIXME: Wrong and probably needs a # prefix
 define void @call__arm_sc_memset(i64 noundef %n) #0 {
 ; CHECK-LABEL: "#call__arm_sc_memset":
-; CHECK: bl __arm_sc_memset
+; CHECK: bl "#__arm_sc_memset"
   tail call void @llvm.memset.p0.i64(ptr align 1 @dst, i8 2, i64 %n, i1 false)
   ret void
 }

[arsenm]

Merge activity

• Jun 27, 8:33 AM UTC: A user started a stack merge that includes this pull request via Graphite.
• Jun 27, 8:50 AM UTC: Graphite rebased this pull request as part of a merge.
• Jun 27, 8:53 AM UTC: @arsenm merged this pull request with Graphite.