From 93d2dd9ccd9ad2b3954c54809be7d368a625a210 Mon Sep 17 00:00:00 2001
From: Pierre Habouzit <phabouzit@apple.com>
Date: Thu, 13 Jul 2017 14:51:43 -0700
Subject: [PATCH] Remove dependency on sys_membarrier on linux

By not inlining the fastpath of dispatch_once() when the hardware cannot
give us the right semantics.
---
 dispatch/once.h  | 12 ++++++++++++
 src/once.c       | 24 +++++++++++++++++++++---
 src/shims/lock.h | 24 ------------------------
 3 files changed, 33 insertions(+), 27 deletions(-)

diff --git a/dispatch/once.h b/dispatch/once.h
index 68acfe803..37a49506d 100644
--- a/dispatch/once.h
+++ b/dispatch/once.h
@@ -40,6 +40,14 @@ __BEGIN_DECLS
 DISPATCH_SWIFT3_UNAVAILABLE("Use lazily initialized globals instead")
 typedef long dispatch_once_t;
 
+#if defined(__x86_64__) || defined(__i386__) || defined(__s390x__)
+#define DISPATCH_ONCE_INLINE_FASTPATH 1
+#elif defined(__APPLE__)
+#define DISPATCH_ONCE_INLINE_FASTPATH 1
+#else
+#define DISPATCH_ONCE_INLINE_FASTPATH 0
+#endif
+
 /*!
  * @function dispatch_once
  *
@@ -65,6 +73,7 @@ void
 dispatch_once(dispatch_once_t *predicate,
 		DISPATCH_NOESCAPE dispatch_block_t block);
 
+#if DISPATCH_ONCE_INLINE_FASTPATH
 DISPATCH_INLINE DISPATCH_ALWAYS_INLINE DISPATCH_NONNULL_ALL DISPATCH_NOTHROW
 DISPATCH_SWIFT3_UNAVAILABLE("Use lazily initialized globals instead")
 void
@@ -81,6 +90,7 @@ _dispatch_once(dispatch_once_t *predicate,
 #undef dispatch_once
 #define dispatch_once _dispatch_once
 #endif
+#endif // DISPATCH_ONCE_INLINE_FASTPATH
 
 API_AVAILABLE(macos(10.6), ios(4.0))
 DISPATCH_EXPORT DISPATCH_NONNULL1 DISPATCH_NONNULL3 DISPATCH_NOTHROW
@@ -89,6 +99,7 @@ void
 dispatch_once_f(dispatch_once_t *predicate, void *_Nullable context,
 		dispatch_function_t function);
 
+#if DISPATCH_ONCE_INLINE_FASTPATH
 DISPATCH_INLINE DISPATCH_ALWAYS_INLINE DISPATCH_NONNULL1 DISPATCH_NONNULL3
 DISPATCH_NOTHROW
 DISPATCH_SWIFT3_UNAVAILABLE("Use lazily initialized globals instead")
@@ -105,6 +116,7 @@ _dispatch_once_f(dispatch_once_t *predicate, void *_Nullable context,
 }
 #undef dispatch_once_f
 #define dispatch_once_f _dispatch_once_f
+#endif // DISPATCH_ONCE_INLINE_FASTPATH
 
 __END_DECLS
 
diff --git a/src/once.c b/src/once.c
index 75d7a39a5..c01538c9d 100644
--- a/src/once.c
+++ b/src/once.c
@@ -40,9 +40,15 @@ dispatch_once(dispatch_once_t *val, dispatch_block_t block)
 }
 #endif
 
-DISPATCH_NOINLINE
-void
-dispatch_once_f(dispatch_once_t *val, void *ctxt, dispatch_function_t func)
+#if DISPATCH_ONCE_INLINE_FASTPATH
+#define DISPATCH_ONCE_SLOW_INLINE inline DISPATCH_ALWAYS_INLINE
+#else
+#define DISPATCH_ONCE_SLOW_INLINE DISPATCH_NOINLINE
+#endif // DISPATCH_ONCE_INLINE_FASTPATH
+
+DISPATCH_ONCE_SLOW_INLINE
+static void
+dispatch_once_f_slow(dispatch_once_t *val, void *ctxt, dispatch_function_t func)
 {
 #if DISPATCH_GATE_USE_FOR_DISPATCH_ONCE
 	dispatch_once_gate_t l = (dispatch_once_gate_t)val;
@@ -95,3 +101,15 @@ dispatch_once_f(dispatch_once_t *val, void *ctxt, dispatch_function_t func)
 	}
 #endif
 }
+
+DISPATCH_NOINLINE
+void
+dispatch_once_f(dispatch_once_t *val, void *ctxt, dispatch_function_t func)
+{
+#if !DISPATCH_ONCE_INLINE_FASTPATH
+	if (likely(os_atomic_load(val, acquire) == DLOCK_ONCE_DONE)) {
+		return;
+	}
+#endif // !DISPATCH_ONCE_INLINE_FASTPATH
+	return dispatch_once_f_slow(val, ctxt, func);
+}
diff --git a/src/shims/lock.h b/src/shims/lock.h
index 99c556370..0c089aafd 100644
--- a/src/shims/lock.h
+++ b/src/shims/lock.h
@@ -59,9 +59,6 @@ _dispatch_lock_owner(dispatch_lock lock_value)
 #elif defined(__linux__)
 
 #include <linux/futex.h>
-#if !defined(__x86_64__) && !defined(__i386__) && !defined(__s390x__)
-#include <linux/membarrier.h>
-#endif
 #include <unistd.h>
 #include <sys/syscall.h>   /* For SYS_xxx definitions */
 
@@ -473,28 +470,7 @@ DISPATCH_ALWAYS_INLINE
 static inline dispatch_once_t
 _dispatch_once_xchg_done(dispatch_once_t *pred)
 {
-#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
-	// On Intel, any load is a load-acquire, so we don't need to be fancy
-	// same for s390x
 	return os_atomic_xchg(pred, DLOCK_ONCE_DONE, release);
-#elif defined(__linux__)
-	if (unlikely(syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0) < 0)) {
-		/*
-		 * sys_membarrier not supported
-		 *
-		 * Ideally we would call DISPATCH_INTERNAL_CRASH() here, but
-		 * due to ordering constraints in internal.h required by Darwin
-		 * the macro is undefined when this header is included.
-		 * Instead, open-code what would be a call to
-		 * _dispatch_hardware_crash() inside DISPATCH_INTERNAL_CRASH().
-		 */
-		__asm__("");
-		__builtin_trap();
-	}
-	return os_atomic_xchg(pred, DLOCK_ONCE_DONE, relaxed);
-#else
-#  error dispatch_once algorithm not available for this port
-#endif
 }
 
 DISPATCH_ALWAYS_INLINE