diff --git a/PATCHES b/PATCHES index 963c8b113..c3d28b330 100644 --- a/PATCHES +++ b/PATCHES @@ -332,3 +332,24 @@ github commits starting with 29bdc2f from [5e8789e] APPLIED rdar://32283666 [3fba60a] APPLIED rdar://32283666 [d6eb245] APPLIED rdar://32283666 +[0b6c22e] APPLIED rdar://33531111 +[5a3c02a] APPLIED rdar://33531111 +[22df1e7] APPLIED rdar://33531111 +[21273de] APPLIED rdar://33531111 +[dc1857c] APPLIED rdar://33531111 +[56f36b6] APPLIED rdar://33531111 +[c87c6bb] APPLIED rdar://33531111 +[b791d23] APPLIED rdar://33531111 +[c2d0c49] APPLIED rdar://33531111 +[1d25040] APPLIED rdar://33531111 +[ab89c6c] APPLIED rdar://33531111 +[e591e7e] APPLIED rdar://33531111 +[ded5bab] APPLIED rdar://33531111 +[ce90d0c] APPLIED rdar://33531111 +[69c8f3e] APPLIED rdar://33531111 +[23a3a84] APPLIED rdar://33531111 +[79b7529] APPLIED rdar://33531111 +[f8e71eb] APPLIED rdar://33531111 +[8947dcf] APPLIED rdar://33531111 +[5ad9208] APPLIED rdar://33531111 +[698d085] APPLIED rdar://33531111 diff --git a/dispatch/queue.h b/dispatch/queue.h index 606bd30e8..8dab75f9d 100644 --- a/dispatch/queue.h +++ b/dispatch/queue.h @@ -223,7 +223,7 @@ dispatch_sync_f(dispatch_queue_t queue, * @abstract * Constant to pass to dispatch_apply() or dispatch_apply_f() to request that * the system automatically use worker threads that match the configuration of - * the current thread most closely. + * the current thread as closely as possible. * * @discussion * When submitting a block for parallel invocation, passing this constant as the diff --git a/libdispatch.xcodeproj/project.pbxproj b/libdispatch.xcodeproj/project.pbxproj index 361994ff9..e1366476d 100644 --- a/libdispatch.xcodeproj/project.pbxproj +++ b/libdispatch.xcodeproj/project.pbxproj @@ -741,10 +741,12 @@ 96DF70BD0F38FE3C0074BD99 /* once.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; lineEnding = 0; path = once.c; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.c; }; B63B793F1E8F004F0060C1E1 /* dispatch_no_blocks.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = dispatch_no_blocks.c; sourceTree = ""; }; B68330BC1EBCF6080003E71C /* dispatch_wl.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = dispatch_wl.c; sourceTree = ""; }; + B69878521F06F8790088F94F /* dispatch_signals.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = dispatch_signals.c; sourceTree = ""; }; B6AC73FD1EB10973009FB2F2 /* perf_thread_request.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = perf_thread_request.c; sourceTree = ""; }; B6AE9A4A1D7F53B300AC007F /* dispatch_queue_create.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = dispatch_queue_create.c; sourceTree = ""; }; B6AE9A561D7F53C100AC007F /* perf_async_bench.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = perf_async_bench.m; sourceTree = ""; }; B6AE9A581D7F53CB00AC007F /* perf_bench.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = perf_bench.m; sourceTree = ""; }; + B6FA01801F0AD522004479BF /* dispatch_pthread_root_queue.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = dispatch_pthread_root_queue.c; sourceTree = ""; }; C00B0E0A1C5AEBBE000330B3 /* libdispatch_dyld_stub.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libdispatch_dyld_stub.a; sourceTree = BUILT_PRODUCTS_DIR; }; C00B0E121C5AEBF7000330B3 /* libdispatch-dyld-stub.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; path = "libdispatch-dyld-stub.xcconfig"; sourceTree = ""; }; C01866BD1C5973210040FC07 /* libdispatch.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libdispatch.a; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -1060,6 +1062,7 @@ 6E67D9151C1768B300FC98AC /* dispatch_pingpong.c */, 6E326B441C239B61002A6505 /* dispatch_priority.c */, 6E326AB51C225477002A6505 /* dispatch_proc.c */, + B6FA01801F0AD522004479BF /* dispatch_pthread_root_queue.c */, 6E326AB31C224870002A6505 /* dispatch_qos.c */, B6AE9A4A1D7F53B300AC007F /* dispatch_queue_create.c */, 6E67D9111C17669C00FC98AC /* dispatch_queue_finalizer.c */, @@ -1069,6 +1072,7 @@ 6E326ADC1C234396002A6505 /* dispatch_readsync.c */, 6E8E4E6D1C1A35EE0004F5CC /* dispatch_select.c */, 6E8E4E9B1C1A4EF10004F5CC /* dispatch_sema.c */, + B69878521F06F8790088F94F /* dispatch_signals.c */, 6EA2CB841C005DEF0076794A /* dispatch_source.c */, 6E326AE01C234780002A6505 /* dispatch_starfish.c */, 6EE89F3D1BFAF5B000EB140D /* dispatch_state_machine.c */, diff --git a/man/dispatch_apply.3 b/man/dispatch_apply.3 index 5a43a0a13..57c99a8a7 100644 --- a/man/dispatch_apply.3 +++ b/man/dispatch_apply.3 @@ -1,4 +1,4 @@ -.\" Copyright (c) 2008-2010 Apple Inc. All rights reserved. +.\" Copyright (c) 2008-2017 Apple Inc. All rights reserved. .Dd May 1, 2009 .Dt dispatch_apply 3 .Os Darwin @@ -20,21 +20,32 @@ The .Fn dispatch_apply function provides data-level concurrency through a "for (;;)" loop like primitive: .Bd -literal -dispatch_queue_t the_queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0); size_t iterations = 10; // 'idx' is zero indexed, just like: // for (idx = 0; idx < iterations; idx++) -dispatch_apply(iterations, the_queue, ^(size_t idx) { +dispatch_apply(iterations, DISPATCH_APPLY_AUTO, ^(size_t idx) { printf("%zu\\n", idx); }); .Ed .Pp +Although any queue can be used, it is strongly recommended to use +.Vt DISPATCH_APPLY_AUTO +as the +.Vt queue +argument to both +.Fn dispatch_apply +and +.Fn dispatch_apply_f , +as shown in the example above, since this allows the system to automatically use worker threads +that match the configuration of the current thread as closely as possible. +No assumptions should be made about which global concurrent queue will be used. +.Pp Like a "for (;;)" loop, the .Fn dispatch_apply function is synchronous. -If asynchronous behavior is desired, please wrap the call to +If asynchronous behavior is desired, wrap the call to .Fn dispatch_apply with a call to .Fn dispatch_async @@ -49,7 +60,7 @@ achieved (perhaps using a power of two search): .Bd -literal #define STRIDE 3 -dispatch_apply(count / STRIDE, queue, ^(size_t idx) { +dispatch_apply(count / STRIDE, DISPATCH_APPLY_AUTO, ^(size_t idx) { size_t j = idx * STRIDE; size_t j_stop = j + STRIDE; do { @@ -74,12 +85,21 @@ This is in contrast to asynchronous functions which must retain both the block and target queue for the duration of the asynchronous operation (as the calling function may immediately release its interest in these objects). .Sh FUNDAMENTALS -Conceptually, .Fn dispatch_apply -is a convenient wrapper around +and +.Fn dispatch_apply_f +attempt to quickly create enough worker threads to efficiently iterate work in parallel. +By contrast, a loop that passes work items individually to .Fn dispatch_async -and a semaphore to wait for completion. -In practice, the dispatch library optimizes this function. +or +.Fn dispatch_async_f +will incur more overhead and does not express the desired parallel execution semantics to +the system, so may not create an optimal number of worker threads for a parallel workload. +For this reason, prefer to use +.Fn dispatch_apply +or +.Fn dispatch_apply_f +when parallel execution is important. .Pp The .Fn dispatch_apply @@ -99,5 +119,4 @@ use a for-loop around invocations of .Sh SEE ALSO .Xr dispatch 3 , .Xr dispatch_async 3 , -.Xr dispatch_queue_create 3 , -.Xr dispatch_semaphore_create 3 +.Xr dispatch_queue_create 3 diff --git a/os/voucher_private.h b/os/voucher_private.h index 6675a0edb..aecbbc9ff 100644 --- a/os/voucher_private.h +++ b/os/voucher_private.h @@ -202,8 +202,23 @@ voucher_decrement_importance_count4CF(voucher_t _Nullable voucher); * This flag is ignored if a specific voucher object is assigned with the * dispatch_block_create_with_voucher* functions, and is equivalent to passing * the NULL voucher to these functions. + * + * @const DISPATCH_BLOCK_IF_LAST_RESET_QUEUE_QOS_OVERRIDE + * Flag indicating that this dispatch block object should try to reset the + * recorded maximum QoS of all currently enqueued items on a serial dispatch + * queue at the base of a queue hierarchy. + * + * This is only works if the queue becomes empty by dequeuing the block in + * question, and then allows that block to enqueue more work on this hierarchy + * without perpetuating QoS overrides resulting from items previously executed + * on the hierarchy. + * + * A dispatch block object created with this flag set cannot be used with + * dispatch_block_wait() or dispatch_block_cancel(). */ -#define DISPATCH_BLOCK_NO_VOUCHER (0x40) +#define DISPATCH_BLOCK_NO_VOUCHER (0x40ul) + +#define DISPATCH_BLOCK_IF_LAST_RESET_QUEUE_QOS_OVERRIDE (0x80ul) /*! * @function dispatch_block_create_with_voucher diff --git a/private/private.h b/private/private.h index cc9d57842..ed9f876cc 100644 --- a/private/private.h +++ b/private/private.h @@ -43,6 +43,9 @@ #include #endif #include +#if TARGET_OS_MAC +#include +#endif #ifndef __DISPATCH_BUILDING_DISPATCH__ #include diff --git a/private/source_private.h b/private/source_private.h index 019f648a6..ad22e6a6a 100644 --- a/private/source_private.h +++ b/private/source_private.h @@ -102,6 +102,13 @@ DISPATCH_SOURCE_TYPE_DECL(memorystatus); API_AVAILABLE(macos(10.8), ios(6.0)) DISPATCH_LINUX_UNAVAILABLE() DISPATCH_SOURCE_TYPE_DECL(sock); +/*! + * @const DISPATCH_SOURCE_TYPE_NW_CHANNEL + * @discussion A dispatch source that monitors events on a network channel. + */ +#define DISPATCH_SOURCE_TYPE_NW_CHANNEL (&_dispatch_source_type_nw_channel) +API_AVAILABLE(macos(10.12), ios(10.0), tvos(10.0), watchos(3.0)) DISPATCH_LINUX_UNAVAILABLE() +DISPATCH_SOURCE_TYPE_DECL(nw_channel); __END_DECLS @@ -165,6 +172,15 @@ enum { DISPATCH_SOCK_NOTIFY_ACK = 0x00004000, }; +/*! + * @enum dispatch_source_nw_channel_flags_t + * + * @constant DISPATCH_NW_CHANNEL_FLOW_ADV_UPDATE + * Received network channel flow advisory. + */ +enum { + DISPATCH_NW_CHANNEL_FLOW_ADV_UPDATE = 0x00000001, +}; /*! * @enum dispatch_source_vfs_flags_t diff --git a/src/apply.c b/src/apply.c index 9d6452225..6f44cf90b 100644 --- a/src/apply.c +++ b/src/apply.c @@ -253,12 +253,23 @@ dispatch_apply_f(size_t iterations, dispatch_queue_t dq, void *ctxt, if (unlikely(iterations == 0)) { return; } - int32_t thr_cnt = (int32_t)dispatch_hw_config(active_cpus); dispatch_thread_context_t dtctxt = _dispatch_thread_context_find(_dispatch_apply_key); size_t nested = dtctxt ? dtctxt->dtc_apply_nesting : 0; dispatch_queue_t old_dq = _dispatch_queue_get_current(); + if (likely(dq == DISPATCH_APPLY_AUTO)) { + dq = _dispatch_apply_root_queue(old_dq); + } + dispatch_qos_t qos = _dispatch_priority_qos(dq->dq_priority); + if (unlikely(dq->do_targetq)) { + // if the queue passed-in is not a root queue, use the current QoS + // since the caller participates in the work anyway + qos = _dispatch_qos_from_pp(_dispatch_get_priority()); + } + int32_t thr_cnt = (int32_t)_dispatch_qos_max_parallelism(qos, + DISPATCH_MAX_PARALLELISM_ACTIVE); + if (likely(!nested)) { nested = iterations; } else { @@ -269,9 +280,6 @@ dispatch_apply_f(size_t iterations, dispatch_queue_t dq, void *ctxt, if (iterations < (size_t)thr_cnt) { thr_cnt = (int32_t)iterations; } - if (likely(dq == DISPATCH_APPLY_AUTO)) { - dq = _dispatch_apply_root_queue(old_dq); - } struct dispatch_continuation_s dc = { .dc_func = (void*)func, .dc_ctxt = ctxt, diff --git a/src/event/event_config.h b/src/event/event_config.h index 2ac3c428d..60f776f95 100644 --- a/src/event/event_config.h +++ b/src/event/event_config.h @@ -50,6 +50,25 @@ #define DISPATCH_MACHPORT_DEBUG 0 #endif +#ifndef DISPATCH_TIMER_ASSERTIONS +#if DISPATCH_DEBUG +#define DISPATCH_TIMER_ASSERTIONS 1 +#else +#define DISPATCH_TIMER_ASSERTIONS 0 +#endif +#endif + +#if DISPATCH_TIMER_ASSERTIONS +#define DISPATCH_TIMER_ASSERT(a, op, b, text) ({ \ + typeof(a) _a = (a); \ + if (unlikely(!(_a op (b)))) { \ + DISPATCH_CLIENT_CRASH(_a, "Timer: " text); \ + } \ + }) +#else +#define DISPATCH_TIMER_ASSERT(a, op, b, text) ((void)0) +#endif + #ifndef EV_VANISHED #define EV_VANISHED 0x0200 #endif @@ -105,6 +124,11 @@ # ifndef VQ_DESIRED_DISK # undef HAVE_DECL_VQ_DESIRED_DISK # endif // VQ_DESIRED_DISK + +# if !defined(EVFILT_NW_CHANNEL) && defined(__APPLE__) +# define EVFILT_NW_CHANNEL (-16) +# define NOTE_FLOW_ADV_UPDATE 0x1 +# endif #else // DISPATCH_EVENT_BACKEND_KEVENT # define EV_ADD 0x0001 # define EV_DELETE 0x0002 diff --git a/src/event/event_kevent.c b/src/event/event_kevent.c index c15a397b4..8fe76d55c 100644 --- a/src/event/event_kevent.c +++ b/src/event/event_kevent.c @@ -671,8 +671,9 @@ _dispatch_kq_drain(dispatch_wlh_t wlh, dispatch_kevent_t ke, int n, r = 0; } else if (flags & KEVENT_FLAG_ERROR_EVENTS) { for (i = 0, r = 0; i < n; i++) { - if ((ke_out[i].flags & EV_ERROR) && (r = (int)ke_out[i].data)) { + if ((ke_out[i].flags & EV_ERROR) && ke_out[i].data) { _dispatch_kevent_drain(&ke_out[i]); + r = (int)ke_out[i].data; } } } else { @@ -1407,6 +1408,17 @@ const dispatch_source_type_s _dispatch_source_type_sock = { }; #endif // EVFILT_SOCK +#ifdef EVFILT_NW_CHANNEL +const dispatch_source_type_s _dispatch_source_type_nw_channel = { + .dst_kind = "nw_channel", + .dst_filter = EVFILT_NW_CHANNEL, + .dst_flags = DISPATCH_EV_DIRECT|EV_CLEAR|EV_VANISHED, + .dst_mask = NOTE_FLOW_ADV_UPDATE, + .dst_size = sizeof(struct dispatch_source_refs_s), + .dst_create = _dispatch_unote_create_with_fd, + .dst_merge_evt = _dispatch_source_merge_evt, +}; +#endif // EVFILT_NW_CHANNEL #if DISPATCH_USE_MEMORYSTATUS @@ -1609,9 +1621,9 @@ _dispatch_mach_notify_source_invoke(mach_msg_header_t *hdr) if (!tlr) { DISPATCH_INTERNAL_CRASH(0, "message received without expected trailer"); } - if (tlr->msgh_audit.val[DISPATCH_MACH_AUDIT_TOKEN_PID] != 0) { - (void)dispatch_assume_zero( - tlr->msgh_audit.val[DISPATCH_MACH_AUDIT_TOKEN_PID]); + if (hdr->msgh_id <= MACH_NOTIFY_LAST + && dispatch_assume_zero(tlr->msgh_audit.val[ + DISPATCH_MACH_AUDIT_TOKEN_PID])) { mach_msg_destroy(hdr); return; } diff --git a/src/init.c b/src/init.c index dea5e8769..6672fac45 100644 --- a/src/init.c +++ b/src/init.c @@ -897,6 +897,7 @@ void _dispatch_temporary_resource_shortage(void) { sleep(1); + asm(""); // prevent tailcall } void * diff --git a/src/inline_internal.h b/src/inline_internal.h index 0ed9e51a8..4103c688f 100644 --- a/src/inline_internal.h +++ b/src/inline_internal.h @@ -1740,7 +1740,7 @@ static inline dispatch_priority_t _dispatch_root_queue_identity_assume(dispatch_queue_t assumed_rq) { dispatch_priority_t old_dbp = _dispatch_get_basepri(); - dispatch_assert(dx_type(assumed_rq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE); + dispatch_assert(dx_hastypeflag(assumed_rq, QUEUE_ROOT)); _dispatch_reset_basepri(assumed_rq->dq_priority); _dispatch_queue_set_current(assumed_rq); return old_dbp; @@ -2108,11 +2108,25 @@ _dispatch_queue_priority_inherit_from_target(dispatch_queue_t dq, if ((!_dispatch_priority_qos(pri) || (pri & inherited_flag)) && (tpri & rootqueue_flag)) { + if (_dispatch_priority_override_qos(pri) == DISPATCH_QOS_SATURATED) { + pri &= DISPATCH_PRIORITY_OVERRIDE_MASK; + } else { + pri = 0; + } if (tpri & defaultqueue_flag) { - dq->dq_priority = 0; + // base queues need to know they target + // the default root queue so that _dispatch_queue_override_qos() + // in _dispatch_queue_class_wakeup() can fallback to QOS_DEFAULT + // if no other priority was provided. + pri |= defaultqueue_flag; } else { - dq->dq_priority = (tpri & ~rootqueue_flag) | inherited_flag; + pri |= (tpri & ~rootqueue_flag) | inherited_flag; } + dq->dq_priority = pri; + } else if (pri & defaultqueue_flag) { + // the DEFAULTQUEUE flag is only set on queues due to the code above, + // and must never be kept if we don't target a global root queue. + dq->dq_priority = (pri & ~defaultqueue_flag); } #else (void)dq; (void)tq; @@ -2272,7 +2286,9 @@ static inline dispatch_qos_t _dispatch_queue_override_qos(dispatch_queue_class_t dqu, dispatch_qos_t qos) { if (dqu._oq->oq_priority & DISPATCH_PRIORITY_FLAG_DEFAULTQUEUE) { - return qos; + // queues targeting the default root queue use any asynchronous + // workitem priority available and fallback to QOS_DEFAULT otherwise. + return qos ? qos : DISPATCH_QOS_DEFAULT; } // for asynchronous workitems, queue priority is the floor for overrides return MAX(qos, _dispatch_priority_qos(dqu._oq->oq_priority)); @@ -2338,14 +2354,20 @@ _dispatch_block_has_private_data(const dispatch_block_t block) } DISPATCH_ALWAYS_INLINE DISPATCH_WARN_RESULT -static inline bool -_dispatch_block_invoke_should_set_priority(dispatch_block_flags_t flags) -{ - if (flags & DISPATCH_BLOCK_HAS_PRIORITY) { - return (flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) || - !(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS); - } - return false; +static inline pthread_priority_t +_dispatch_block_invoke_should_set_priority(dispatch_block_flags_t flags, + pthread_priority_t new_pri) +{ + pthread_priority_t old_pri, p = 0; // 0 means do not change priority. + if ((flags & DISPATCH_BLOCK_HAS_PRIORITY) + && ((flags & DISPATCH_BLOCK_ENFORCE_QOS_CLASS) || + !(flags & DISPATCH_BLOCK_INHERIT_QOS_CLASS))) { + old_pri = _dispatch_get_priority(); + new_pri &= ~_PTHREAD_PRIORITY_FLAGS_MASK; + p = old_pri & ~_PTHREAD_PRIORITY_FLAGS_MASK; + if (!p || p >= new_pri) p = 0; + } + return p; } DISPATCH_ALWAYS_INLINE diff --git a/src/internal.h b/src/internal.h index 3f481aad7..84e33e350 100644 --- a/src/internal.h +++ b/src/internal.h @@ -459,9 +459,9 @@ void _dispatch_log(const char *msg, ...); if (__builtin_constant_p(e)) { \ dispatch_static_assert(e); \ } else { \ - typeof(e) _e = fastpath(e); /* always eval 'e' */ \ - if (!_e) { \ - __assert_rtn(__func__, __FILE__, __LINE__, #e); \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && !_e)) { \ + _dispatch_abort(__LINE__, (long)_e); \ } \ } \ } while (0) @@ -483,9 +483,9 @@ _dispatch_assert(long e, size_t line) if (__builtin_constant_p(e)) { \ dispatch_static_assert(e); \ } else { \ - typeof(e) _e = slowpath(e); /* always eval 'e' */ \ - if (_e) { \ - __assert_rtn(__func__, __FILE__, __LINE__, #e); \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && _e)) { \ + _dispatch_abort(__LINE__, (long)_e); \ } \ } \ } while (0) @@ -506,8 +506,8 @@ _dispatch_assert_zero(long e, size_t line) */ #if __GNUC__ #define dispatch_assume(e) ({ \ - typeof(e) _e = fastpath(e); /* always eval 'e' */ \ - if (!_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(!_e)) { \ if (__builtin_constant_p(e)) { \ dispatch_static_assert(e); \ } \ @@ -531,8 +531,8 @@ _dispatch_assume(long e, long line) */ #if __GNUC__ #define dispatch_assume_zero(e) ({ \ - typeof(e) _e = slowpath(e); /* always eval 'e' */ \ - if (_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(_e)) { \ if (__builtin_constant_p(e)) { \ dispatch_static_assert(e); \ } \ @@ -558,8 +558,8 @@ _dispatch_assume_zero(long e, long line) if (__builtin_constant_p(e)) { \ dispatch_static_assert(e); \ } else { \ - typeof(e) _e = fastpath(e); /* always eval 'e' */ \ - if (DISPATCH_DEBUG && !_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && !_e)) { \ _dispatch_log("%s() 0x%lx: " msg, __func__, (long)_e, ##args); \ abort(); \ } \ @@ -567,8 +567,8 @@ _dispatch_assume_zero(long e, long line) } while (0) #else #define dispatch_debug_assert(e, msg, args...) do { \ - long _e = (long)fastpath(e); /* always eval 'e' */ \ - if (DISPATCH_DEBUG && !_e) { \ + typeof(e) _e = (e); /* always eval 'e' */ \ + if (unlikely(DISPATCH_DEBUG && !_e)) { \ _dispatch_log("%s() 0x%lx: " msg, __FUNCTION__, _e, ##args); \ abort(); \ } \ @@ -626,7 +626,7 @@ DISPATCH_ALWAYS_INLINE static inline void _dispatch_fork_becomes_unsafe(void) { - if (!fastpath(_dispatch_is_multithreaded_inline())) { + if (unlikely(!_dispatch_is_multithreaded_inline())) { _dispatch_fork_becomes_unsafe_slow(); DISPATCH_COMPILER_CAN_ASSUME(_dispatch_is_multithreaded_inline()); } @@ -732,6 +732,20 @@ extern bool _dispatch_memory_warn; #endif // HAVE_SYS_GUARDED_H +#if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION +typedef struct dispatch_trace_timer_params_s { + int64_t deadline, interval, leeway; +} *dispatch_trace_timer_params_t; + +#ifdef __cplusplus +extern "C++" { +#endif +#include "provider.h" +#ifdef __cplusplus +} +#endif +#endif // DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION + #if __has_include() #include #ifndef DBG_DISPATCH @@ -754,7 +768,7 @@ extern bool _dispatch_memory_warn; #define ARIADNE_ENTER_DISPATCH_MAIN_CODE 0 #endif #if !defined(DISPATCH_USE_VOUCHER_KDEBUG_TRACE) && \ - (DISPATCH_INTROSPECTION || DISPATCH_PROFILE) + (DISPATCH_INTROSPECTION || DISPATCH_PROFILE || DISPATCH_DEBUG) #define DISPATCH_USE_VOUCHER_KDEBUG_TRACE 1 #endif diff --git a/src/mach.c b/src/mach.c index 0f9e9a8f3..699492da0 100644 --- a/src/mach.c +++ b/src/mach.c @@ -59,7 +59,7 @@ static inline mach_msg_header_t* _dispatch_mach_msg_get_msg( static void _dispatch_mach_send_push(dispatch_mach_t dm, dispatch_object_t dou, dispatch_qos_t qos); static void _dispatch_mach_cancel(dispatch_mach_t dm); -static void _dispatch_mach_send_barrier_drain_push(dispatch_mach_t dm, +static void _dispatch_mach_push_send_barrier_drain(dispatch_mach_t dm, dispatch_qos_t qos); static void _dispatch_mach_handle_or_push_received_msg(dispatch_mach_t dm, dispatch_mach_msg_t dmsg); @@ -73,6 +73,9 @@ static void _dispatch_mach_notification_kevent_unregister(dispatch_mach_t dm); static void _dispatch_mach_notification_kevent_register(dispatch_mach_t dm, mach_port_t send); +// For tests only. +DISPATCH_EXPORT void _dispatch_mach_hooks_install_default(void); + dispatch_source_t _dispatch_source_create_mach_msg_direct_recv(mach_port_t recvp, const struct dispatch_continuation_s *dc) @@ -153,6 +156,13 @@ dispatch_mach_hooks_install_4libxpc(dispatch_mach_xpc_hooks_t hooks) } } +void +_dispatch_mach_hooks_install_default(void) +{ + os_atomic_store(&_dispatch_mach_xpc_hooks, + &_dispatch_mach_xpc_hooks_default, relaxed); +} + #pragma mark - #pragma mark dispatch_mach_t @@ -431,6 +441,9 @@ _dispatch_mach_reply_kevent_register(dispatch_mach_t dm, mach_port_t reply_port, if (!drq) { pri = dm->dq_priority; wlh = dm->dm_recv_refs->du_wlh; + } else if (dx_type(drq) == DISPATCH_QUEUE_NETWORK_EVENT_TYPE) { + pri = DISPATCH_PRIORITY_FLAG_MANAGER; + wlh = (dispatch_wlh_t)drq; } else if (dx_hastypeflag(drq, QUEUE_ROOT)) { pri = drq->dq_priority; wlh = DISPATCH_WLH_ANON; @@ -1386,7 +1399,7 @@ _dispatch_mach_send_drain(dispatch_mach_t dm, dispatch_invoke_flags_t flags, if (new_state & DISPATCH_MACH_STATE_PENDING_BARRIER) { qos = _dmsr_state_max_qos(new_state); - _dispatch_mach_send_barrier_drain_push(dm, qos); + _dispatch_mach_push_send_barrier_drain(dm, qos); } else { if (needs_mgr || dm->dm_needs_mgr) { qos = _dmsr_state_max_qos(new_state); @@ -1472,7 +1485,7 @@ _dispatch_mach_send_barrier_drain_invoke(dispatch_continuation_t dc, DISPATCH_NOINLINE static void -_dispatch_mach_send_barrier_drain_push(dispatch_mach_t dm, dispatch_qos_t qos) +_dispatch_mach_push_send_barrier_drain(dispatch_mach_t dm, dispatch_qos_t qos) { dispatch_continuation_t dc = _dispatch_continuation_alloc(); @@ -1534,7 +1547,7 @@ _dispatch_mach_send_push(dispatch_mach_t dm, dispatch_continuation_t dc, dispatch_wakeup_flags_t wflags = 0; if (state_flags & DISPATCH_MACH_STATE_PENDING_BARRIER) { - _dispatch_mach_send_barrier_drain_push(dm, qos); + _dispatch_mach_push_send_barrier_drain(dm, qos); } else if (wakeup || dmsr->dmsr_disconnect_cnt || (dm->dq_atomic_flags & DSF_CANCELED)) { wflags = DISPATCH_WAKEUP_MAKE_DIRTY | DISPATCH_WAKEUP_CONSUME_2; @@ -1739,6 +1752,7 @@ _dispatch_mach_priority_propagate(mach_msg_option_t options, } *msg_pp = _dispatch_priority_compute_propagated(0, flags); // TODO: remove QoS contribution of sync IPC messages to send queue + // rdar://31848737 return _dispatch_qos_from_pp(*msg_pp); } @@ -2216,7 +2230,7 @@ dispatch_mach_send_barrier_f(dispatch_mach_t dm, void *context, dispatch_function_t func) { dispatch_continuation_t dc = _dispatch_continuation_alloc(); - uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT; + uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER; dispatch_qos_t qos; _dispatch_continuation_init_f(dc, dm, context, func, 0, 0, dc_flags); @@ -2231,7 +2245,7 @@ void dispatch_mach_send_barrier(dispatch_mach_t dm, dispatch_block_t barrier) { dispatch_continuation_t dc = _dispatch_continuation_alloc(); - uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT; + uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER; dispatch_qos_t qos; _dispatch_continuation_init(dc, dm, barrier, 0, 0, dc_flags); @@ -2247,7 +2261,7 @@ dispatch_mach_receive_barrier_f(dispatch_mach_t dm, void *context, dispatch_function_t func) { dispatch_continuation_t dc = _dispatch_continuation_alloc(); - uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT; + uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER; _dispatch_continuation_init_f(dc, dm, context, func, 0, 0, dc_flags); _dispatch_mach_barrier_set_vtable(dc, dm, DC_VTABLE(MACH_RECV_BARRIER)); @@ -2259,7 +2273,7 @@ void dispatch_mach_receive_barrier(dispatch_mach_t dm, dispatch_block_t barrier) { dispatch_continuation_t dc = _dispatch_continuation_alloc(); - uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT; + uintptr_t dc_flags = DISPATCH_OBJ_CONSUME_BIT | DISPATCH_OBJ_MACH_BARRIER; _dispatch_continuation_init(dc, dm, barrier, 0, 0, dc_flags); _dispatch_mach_barrier_set_vtable(dc, dm, DC_VTABLE(MACH_RECV_BARRIER)); diff --git a/src/object.c b/src/object.c index 43f580bd2..86d100507 100644 --- a/src/object.c +++ b/src/object.c @@ -236,8 +236,9 @@ void * dispatch_get_context(dispatch_object_t dou) { DISPATCH_OBJECT_TFB(_dispatch_objc_get_context, dou); - if (slowpath(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) || - slowpath(dx_hastypeflag(dou._do, QUEUE_ROOT))) { + if (unlikely(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT || + dx_hastypeflag(dou._do, QUEUE_ROOT) || + dx_hastypeflag(dou._do, QUEUE_BASE))) { return NULL; } return dou._do->do_ctxt; @@ -247,8 +248,9 @@ void dispatch_set_context(dispatch_object_t dou, void *context) { DISPATCH_OBJECT_TFB(_dispatch_objc_set_context, dou, context); - if (slowpath(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) || - slowpath(dx_hastypeflag(dou._do, QUEUE_ROOT))) { + if (unlikely(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT || + dx_hastypeflag(dou._do, QUEUE_ROOT) || + dx_hastypeflag(dou._do, QUEUE_BASE))) { return; } dou._do->do_ctxt = context; @@ -258,8 +260,9 @@ void dispatch_set_finalizer_f(dispatch_object_t dou, dispatch_function_t finalizer) { DISPATCH_OBJECT_TFB(_dispatch_objc_set_finalizer_f, dou, finalizer); - if (slowpath(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) || - slowpath(dx_hastypeflag(dou._do, QUEUE_ROOT))) { + if (unlikely(dou._do->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT || + dx_hastypeflag(dou._do, QUEUE_ROOT) || + dx_hastypeflag(dou._do, QUEUE_BASE))) { return; } dou._do->do_finalizer = finalizer; @@ -271,8 +274,9 @@ dispatch_set_target_queue(dispatch_object_t dou, dispatch_queue_t tq) DISPATCH_OBJECT_TFB(_dispatch_objc_set_target_queue, dou, tq); if (dx_vtable(dou._do)->do_set_targetq) { dx_vtable(dou._do)->do_set_targetq(dou._do, tq); - } else if (dou._do->do_ref_cnt != DISPATCH_OBJECT_GLOBAL_REFCNT && - !slowpath(dx_hastypeflag(dou._do, QUEUE_ROOT))) { + } else if (likely(dou._do->do_ref_cnt != DISPATCH_OBJECT_GLOBAL_REFCNT && + !dx_hastypeflag(dou._do, QUEUE_ROOT) && + !dx_hastypeflag(dou._do, QUEUE_BASE))) { if (slowpath(!tq)) { tq = _dispatch_get_root_queue(DISPATCH_QOS_DEFAULT, false); } diff --git a/src/object.m b/src/object.m index cc97cc3db..efee82947 100644 --- a/src/object.m +++ b/src/object.m @@ -387,6 +387,14 @@ @implementation OS_OBJECT_CLASS(voucher) DISPATCH_UNAVAILABLE_INIT() DISPATCH_OBJC_LOAD() +-(id)retain { + return (id)_voucher_retain_inline((struct voucher_s *)self); +} + +-(oneway void)release { + return _voucher_release_inline((struct voucher_s *)self); +} + - (void)_xref_dispose { return _voucher_xref_dispose(self); // calls _os_object_release_internal() } diff --git a/src/object_internal.h b/src/object_internal.h index 0060f27f8..4504f6587 100644 --- a/src/object_internal.h +++ b/src/object_internal.h @@ -355,6 +355,8 @@ enum { _DISPATCH_DISK_TYPE = 0x70000, // meta-type for io disks _DISPATCH_QUEUE_ROOT_TYPEFLAG = 0x0100, // bit set for any root queues + _DISPATCH_QUEUE_BASE_TYPEFLAG = 0x0200, // base of a hierarchy + // targets a root queue #define DISPATCH_CONTINUATION_TYPE(name) \ (_DISPATCH_CONTINUATION_TYPE | DC_##name##_TYPE) @@ -372,10 +374,11 @@ enum { DISPATCH_QUEUE_GLOBAL_ROOT_TYPE = 4 | _DISPATCH_QUEUE_TYPE | _DISPATCH_QUEUE_ROOT_TYPEFLAG, DISPATCH_QUEUE_NETWORK_EVENT_TYPE = 5 | _DISPATCH_QUEUE_TYPE | - _DISPATCH_QUEUE_ROOT_TYPEFLAG, + _DISPATCH_QUEUE_BASE_TYPEFLAG, DISPATCH_QUEUE_RUNLOOP_TYPE = 6 | _DISPATCH_QUEUE_TYPE | - _DISPATCH_QUEUE_ROOT_TYPEFLAG, - DISPATCH_QUEUE_MGR_TYPE = 7 | _DISPATCH_QUEUE_TYPE, + _DISPATCH_QUEUE_BASE_TYPEFLAG, + DISPATCH_QUEUE_MGR_TYPE = 7 | _DISPATCH_QUEUE_TYPE | + _DISPATCH_QUEUE_BASE_TYPEFLAG, DISPATCH_QUEUE_SPECIFIC_TYPE = 8 | _DISPATCH_QUEUE_TYPE, DISPATCH_SEMAPHORE_TYPE = 1 | _DISPATCH_SEMAPHORE_TYPE, diff --git a/src/provider.d b/src/provider.d index ede3c56b3..13bcf7a93 100644 --- a/src/provider.d +++ b/src/provider.d @@ -101,3 +101,41 @@ provider dispatch { #pragma D attributes Private/Private/Common provider dispatch function #pragma D attributes Evolving/Evolving/Common provider dispatch name #pragma D attributes Evolving/Evolving/Common provider dispatch args + +typedef struct voucher_s *voucher_t; + +/* + * Probes for vouchers + */ +provider voucher { + + /* + * Voucher lifetime: + * + * voucher$target:::create A new voucher is being created + * voucher$target:::dispose A voucher is being freed + * voucher$target:::retain A voucher is being retained + * voucher$target:::release A voucher is being released + */ + probe create(voucher_t voucher, mach_port_t kv, uint64_t activity_id); + probe dispose(voucher_t voucher); + probe retain(voucher_t voucher, int resulting_refcnt); + probe release(voucher_t voucher, int resulting_refcnt); + + /* + * Thread adoption + * + * voucher$target:::adopt A voucher is being adopted by the current thread + * voucher$target:::orphan A voucher is being orphanned by the current thread + */ + probe adopt(voucher_t voucher); + probe orphan(voucher_t voucher); + +}; + +#pragma D attributes Evolving/Evolving/Common provider voucher provider +#pragma D attributes Private/Private/Common provider voucher module +#pragma D attributes Private/Private/Common provider voucher function +#pragma D attributes Evolving/Evolving/Common provider voucher name +#pragma D attributes Evolving/Evolving/Common provider voucher args + diff --git a/src/queue.c b/src/queue.c index fe26ab71d..33e500957 100644 --- a/src/queue.c +++ b/src/queue.c @@ -26,20 +26,32 @@ #if HAVE_PTHREAD_WORKQUEUES || DISPATCH_USE_INTERNAL_WORKQUEUE #define DISPATCH_USE_WORKQUEUES 1 #endif -#if (!HAVE_PTHREAD_WORKQUEUES || DISPATCH_DEBUG) && !defined(DISPATCH_ENABLE_THREAD_POOL) +#if (!HAVE_PTHREAD_WORKQUEUES || DISPATCH_DEBUG) && \ + !defined(DISPATCH_ENABLE_THREAD_POOL) #define DISPATCH_ENABLE_THREAD_POOL 1 #endif #if DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES || DISPATCH_ENABLE_THREAD_POOL #define DISPATCH_USE_PTHREAD_POOL 1 #endif -#if HAVE_PTHREAD_WORKQUEUES && (!HAVE_PTHREAD_WORKQUEUE_QOS || DISPATCH_DEBUG) && \ - !HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP && \ +#if HAVE_PTHREAD_WORKQUEUES && (!HAVE_PTHREAD_WORKQUEUE_QOS || \ + DISPATCH_DEBUG) && !HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP && \ !defined(DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK) #define DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK 1 #endif -#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP || DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK +#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP && (DISPATCH_DEBUG || \ + (!DISPATCH_USE_KEVENT_WORKQUEUE && !HAVE_PTHREAD_WORKQUEUE_QOS)) && \ + !defined(DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP) +#define DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP 1 +#endif +#if DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP || \ + DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK || \ + DISPATCH_USE_INTERNAL_WORKQUEUE +#if !DISPATCH_USE_INTERNAL_WORKQUEUE #define DISPATCH_USE_WORKQ_PRIORITY 1 #endif +#define DISPATCH_USE_WORKQ_OPTIONS 1 +#endif + #if DISPATCH_USE_WORKQUEUES && DISPATCH_USE_PTHREAD_POOL && \ !DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK #define pthread_workqueue_t void* @@ -69,7 +81,7 @@ static void _dispatch_worker_thread4(void *context); #if HAVE_PTHREAD_WORKQUEUE_QOS static void _dispatch_worker_thread3(pthread_priority_t priority); #endif -#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#if DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP static void _dispatch_worker_thread2(int priority, int options, void *context); #endif #endif @@ -164,7 +176,9 @@ struct dispatch_root_queue_context_s { #if DISPATCH_USE_WORKQ_PRIORITY int dgq_wq_priority; #endif +#if DISPATCH_USE_WORKQ_OPTIONS int dgq_wq_options; +#endif #if DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK || DISPATCH_USE_PTHREAD_POOL pthread_workqueue_t dgq_kworkqueue; #endif @@ -195,8 +209,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = 0, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS], @@ -208,8 +224,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_BG_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_MAINTENANCE_QOS_OVERCOMMIT], @@ -221,8 +239,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_BG_PRIOQUEUE_CONDITIONAL, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = 0, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS], @@ -234,8 +254,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_BG_PRIOQUEUE_CONDITIONAL, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS_OVERCOMMIT], @@ -247,8 +269,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_LOW_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = 0, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS], @@ -260,8 +284,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_LOW_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_UTILITY_QOS_OVERCOMMIT], @@ -273,8 +299,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_DEFAULT_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = 0, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS], @@ -286,8 +314,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_DEFAULT_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_DEFAULT_QOS_OVERCOMMIT], @@ -299,8 +329,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = 0, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS], @@ -312,8 +344,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT], @@ -325,8 +359,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE_CONDITIONAL, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = 0, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS], @@ -338,8 +374,10 @@ static struct dispatch_root_queue_context_s _dispatch_root_queue_contexts[] = { #if DISPATCH_USE_WORKQ_PRIORITY .dgq_wq_priority = WORKQ_HIGH_PRIOQUEUE_CONDITIONAL, #endif +#if DISPATCH_USE_WORKQ_OPTIONS .dgq_wq_options = WORKQ_ADDTHREADS_OPTION_OVERCOMMIT, #endif +#endif #if DISPATCH_ENABLE_THREAD_POOL .dgq_ctxt = &_dispatch_pthread_root_queue_contexts[ DISPATCH_ROOT_QUEUE_IDX_USER_INTERACTIVE_QOS_OVERCOMMIT], @@ -419,7 +457,7 @@ struct dispatch_queue_s _dispatch_root_queues[] = { ), }; -#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#if DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP static const dispatch_queue_t _dispatch_wq2root_queues[][2] = { [WORKQ_BG_PRIOQUEUE][0] = &_dispatch_root_queues[ DISPATCH_ROOT_QUEUE_IDX_BACKGROUND_QOS], @@ -442,7 +480,7 @@ static const dispatch_queue_t _dispatch_wq2root_queues[][2] = { &_dispatch_root_queues[ DISPATCH_ROOT_QUEUE_IDX_USER_INITIATED_QOS_OVERCOMMIT], }; -#endif // HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#endif // DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP #if DISPATCH_USE_MGR_THREAD && DISPATCH_ENABLE_PTHREAD_ROOT_QUEUES static struct dispatch_queue_s _dispatch_mgr_root_queue; @@ -650,7 +688,7 @@ _dispatch_root_queues_init_workq(int *wq_supported) } } #endif // DISPATCH_USE_KEVENT_WORKQUEUE || HAVE_PTHREAD_WORKQUEUE_QOS -#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#if DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP if (!result && !disable_wq) { pthread_workqueue_setdispatchoffset_np( offsetof(struct dispatch_queue_s, dq_serialnum)); @@ -660,7 +698,7 @@ _dispatch_root_queues_init_workq(int *wq_supported) #endif result = !r; } -#endif // HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#endif // DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP #if DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK || DISPATCH_USE_PTHREAD_POOL if (!result) { #if DISPATCH_USE_LEGACY_WORKQUEUE_FALLBACK @@ -788,7 +826,7 @@ libdispatch_init(void) DISPATCH_ROOT_QUEUE_COUNT); dispatch_assert(countof(_dispatch_root_queue_contexts) == DISPATCH_ROOT_QUEUE_COUNT); -#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#if DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP dispatch_assert(sizeof(_dispatch_wq2root_queues) / sizeof(_dispatch_wq2root_queues[0][0]) == WORKQ_NUM_PRIOQUEUE * 2); @@ -1420,7 +1458,7 @@ _dispatch_queue_destroy(dispatch_queue_t dq, bool *allow_free) uint64_t dq_state = os_atomic_load2o(dq, dq_state, relaxed); uint64_t initial_state = DISPATCH_QUEUE_STATE_INIT_VALUE(dq->dq_width); - if (dx_type(dq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE) { + if (dx_hastypeflag(dq, QUEUE_ROOT)) { initial_state = DISPATCH_ROOT_QUEUE_STATE_INIT_VALUE; } dq_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK; @@ -1799,23 +1837,24 @@ _dispatch_queue_set_width2(void *ctxt) uint32_t tmp; dispatch_queue_t dq = _dispatch_queue_get_current(); - if (w > 0) { - tmp = (unsigned int)w; - } else switch (w) { - case 0: - tmp = 1; - break; - case DISPATCH_QUEUE_WIDTH_MAX_PHYSICAL_CPUS: - tmp = dispatch_hw_config(physical_cpus); - break; - case DISPATCH_QUEUE_WIDTH_ACTIVE_CPUS: - tmp = dispatch_hw_config(active_cpus); - break; - default: - // fall through - case DISPATCH_QUEUE_WIDTH_MAX_LOGICAL_CPUS: - tmp = dispatch_hw_config(logical_cpus); - break; + if (w >= 0) { + tmp = w ? (unsigned int)w : 1; + } else { + dispatch_qos_t qos = _dispatch_qos_from_pp(_dispatch_get_priority()); + switch (w) { + case DISPATCH_QUEUE_WIDTH_MAX_PHYSICAL_CPUS: + tmp = _dispatch_qos_max_parallelism(qos, + DISPATCH_MAX_PARALLELISM_PHYSICAL); + break; + case DISPATCH_QUEUE_WIDTH_ACTIVE_CPUS: + tmp = _dispatch_qos_max_parallelism(qos, + DISPATCH_MAX_PARALLELISM_ACTIVE); + break; + case DISPATCH_QUEUE_WIDTH_MAX_LOGICAL_CPUS: + default: + tmp = _dispatch_qos_max_parallelism(qos, 0); + break; + } } if (tmp > DISPATCH_QUEUE_WIDTH_MAX) { tmp = DISPATCH_QUEUE_WIDTH_MAX; @@ -1832,8 +1871,9 @@ _dispatch_queue_set_width2(void *ctxt) void dispatch_queue_set_width(dispatch_queue_t dq, long width) { - if (slowpath(dq->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT) || - slowpath(dx_hastypeflag(dq, QUEUE_ROOT))) { + if (unlikely(dq->do_ref_cnt == DISPATCH_OBJECT_GLOBAL_REFCNT || + dx_hastypeflag(dq, QUEUE_ROOT) || + dx_hastypeflag(dq, QUEUE_BASE))) { return; } @@ -1848,8 +1888,15 @@ dispatch_queue_set_width(dispatch_queue_t dq, long width) DISPATCH_CLIENT_CRASH(type, "Unexpected dispatch object type"); } - _dispatch_barrier_trysync_or_async_f(dq, (void*)(intptr_t)width, - _dispatch_queue_set_width2); + if (likely((int)width >= 0)) { + _dispatch_barrier_trysync_or_async_f(dq, (void*)(intptr_t)width, + _dispatch_queue_set_width2); + } else { + // The negative width constants need to execute on the queue to + // query the queue QoS + _dispatch_barrier_async_detached_f(dq, (void*)(intptr_t)width, + _dispatch_queue_set_width2); + } } static void @@ -2265,7 +2312,7 @@ dispatch_pthread_root_queue_copy_current(void) { dispatch_queue_t dq = _dispatch_queue_get_current(); if (!dq) return NULL; - while (slowpath(dq->do_targetq)) { + while (unlikely(dq->do_targetq)) { dq = dq->do_targetq; } if (dx_type(dq) != DISPATCH_QUEUE_GLOBAL_ROOT_TYPE || @@ -2911,9 +2958,9 @@ _dispatch_block_invoke_direct(const struct dispatch_block_private_data_s *dbcpd) if (atomic_flags & DBF_CANCELED) goto out; pthread_priority_t op = 0, p = 0; - if (_dispatch_block_invoke_should_set_priority(flags)) { - op = _dispatch_get_priority(); - p = dbpd->dbpd_priority; + op = _dispatch_block_invoke_should_set_priority(flags, dbpd->dbpd_priority); + if (op) { + p = dbpd->dbpd_priority; } voucher_t ov, v = DISPATCH_NO_VOUCHER; if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { @@ -2966,9 +3013,55 @@ _dispatch_block_sync_invoke(void *block) } } -DISPATCH_ALWAYS_INLINE +#if DISPATCH_USE_KEVENT_WORKQUEUE static void -_dispatch_block_async_invoke2(dispatch_block_t b, bool release) +_dispatch_block_async_invoke_reset_max_qos(dispatch_queue_t dq, + dispatch_qos_t qos) +{ + uint64_t old_state, new_state, qos_bits = _dq_state_from_qos(qos); + + // Only dispatch queues can reach this point (as opposed to sources or more + // complex objects) which allows us to handle the DIRTY bit protocol by only + // looking at the tail + dispatch_assert(dx_metatype(dq) == _DISPATCH_QUEUE_TYPE); + +again: + os_atomic_rmw_loop2o(dq, dq_state, old_state, new_state, relaxed, { + dispatch_assert(_dq_state_is_base_wlh(old_state)); + if ((old_state & DISPATCH_QUEUE_MAX_QOS_MASK) <= qos_bits) { + // Nothing to do if the QoS isn't going down + os_atomic_rmw_loop_give_up(return); + } + if (_dq_state_is_dirty(old_state)) { + os_atomic_rmw_loop_give_up({ + // just renew the drain lock with an acquire barrier, to see + // what the enqueuer that set DIRTY has done. + // the xor generates better assembly as DISPATCH_QUEUE_DIRTY + // is already in a register + os_atomic_xor2o(dq, dq_state, DISPATCH_QUEUE_DIRTY, acquire); + if (!dq->dq_items_tail) { + goto again; + } + return; + }); + } + + new_state = old_state; + new_state &= ~DISPATCH_QUEUE_MAX_QOS_MASK; + new_state |= qos_bits; + }); + + _dispatch_deferred_items_get()->ddi_wlh_needs_update = true; + _dispatch_event_loop_drain(KEVENT_FLAG_IMMEDIATE); +} +#endif // DISPATCH_USE_KEVENT_WORKQUEUE + +#define DISPATCH_BLOCK_ASYNC_INVOKE_RELEASE 0x1 +#define DISPATCH_BLOCK_ASYNC_INVOKE_NO_OVERRIDE_RESET 0x2 + +DISPATCH_NOINLINE +static void +_dispatch_block_async_invoke2(dispatch_block_t b, unsigned long invoke_flags) { dispatch_block_private_data_t dbpd = _dispatch_block_get_data(b); unsigned int atomic_flags = dbpd->dbpd_atomic_flags; @@ -2976,6 +3069,19 @@ _dispatch_block_async_invoke2(dispatch_block_t b, bool release) DISPATCH_CLIENT_CRASH(atomic_flags, "A block object may not be both " "run more than once and waited for"); } + +#if DISPATCH_USE_KEVENT_WORKQUEUE + if (unlikely((dbpd->dbpd_flags & + DISPATCH_BLOCK_IF_LAST_RESET_QUEUE_QOS_OVERRIDE) && + !(invoke_flags & DISPATCH_BLOCK_ASYNC_INVOKE_NO_OVERRIDE_RESET))) { + dispatch_queue_t dq = _dispatch_get_current_queue(); + dispatch_qos_t qos = _dispatch_qos_from_pp(_dispatch_get_priority()); + if ((dispatch_wlh_t)dq == _dispatch_get_wlh() && !dq->dq_items_tail) { + _dispatch_block_async_invoke_reset_max_qos(dq, qos); + } + } +#endif // DISPATCH_USE_KEVENT_WORKQUEUE + if (!slowpath(atomic_flags & DBF_CANCELED)) { dbpd->dbpd_block(); } @@ -2984,13 +3090,14 @@ _dispatch_block_async_invoke2(dispatch_block_t b, bool release) dispatch_group_leave(_dbpd_group(dbpd)); } } - os_mpsc_queue_t oq; - oq = os_atomic_xchg2o(dbpd, dbpd_queue, NULL, relaxed); + + os_mpsc_queue_t oq = os_atomic_xchg2o(dbpd, dbpd_queue, NULL, relaxed); if (oq) { // balances dispatch_{,barrier_,group_}async _os_object_release_internal_n_inline(oq->_as_os_obj, 2); } - if (release) { + + if (invoke_flags & DISPATCH_BLOCK_ASYNC_INVOKE_RELEASE) { Block_release(b); } } @@ -2998,20 +3105,35 @@ _dispatch_block_async_invoke2(dispatch_block_t b, bool release) static void _dispatch_block_async_invoke(void *block) { - _dispatch_block_async_invoke2(block, false); + _dispatch_block_async_invoke2(block, 0); } static void _dispatch_block_async_invoke_and_release(void *block) { - _dispatch_block_async_invoke2(block, true); + _dispatch_block_async_invoke2(block, DISPATCH_BLOCK_ASYNC_INVOKE_RELEASE); +} + +static void +_dispatch_block_async_invoke_and_release_mach_barrier(void *block) +{ + _dispatch_block_async_invoke2(block, DISPATCH_BLOCK_ASYNC_INVOKE_RELEASE | + DISPATCH_BLOCK_ASYNC_INVOKE_NO_OVERRIDE_RESET); +} + +DISPATCH_ALWAYS_INLINE +static inline bool +_dispatch_block_supports_wait_and_cancel(dispatch_block_private_data_t dbpd) +{ + return dbpd && !(dbpd->dbpd_flags & + DISPATCH_BLOCK_IF_LAST_RESET_QUEUE_QOS_OVERRIDE); } void dispatch_block_cancel(dispatch_block_t db) { dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); - if (!dbpd) { + if (unlikely(!_dispatch_block_supports_wait_and_cancel(dbpd))) { DISPATCH_CLIENT_CRASH(db, "Invalid block object passed to " "dispatch_block_cancel()"); } @@ -3022,7 +3144,7 @@ long dispatch_block_testcancel(dispatch_block_t db) { dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); - if (!dbpd) { + if (unlikely(!_dispatch_block_supports_wait_and_cancel(dbpd))) { DISPATCH_CLIENT_CRASH(db, "Invalid block object passed to " "dispatch_block_testcancel()"); } @@ -3033,7 +3155,7 @@ long dispatch_block_wait(dispatch_block_t db, dispatch_time_t timeout) { dispatch_block_private_data_t dbpd = _dispatch_block_get_data(db); - if (!dbpd) { + if (unlikely(!_dispatch_block_supports_wait_and_cancel(dbpd))) { DISPATCH_CLIENT_CRASH(db, "Invalid block object passed to " "dispatch_block_wait()"); } @@ -3128,7 +3250,10 @@ _dispatch_continuation_init_slow(dispatch_continuation_t dc, _os_object_retain_internal_n_inline(oq->_as_os_obj, 2); } - if (dc_flags & DISPATCH_OBJ_CONSUME_BIT) { + if (dc_flags & DISPATCH_OBJ_MACH_BARRIER) { + dispatch_assert(dc_flags & DISPATCH_OBJ_CONSUME_BIT); + dc->dc_func = _dispatch_block_async_invoke_and_release_mach_barrier; + } else if (dc_flags & DISPATCH_OBJ_CONSUME_BIT) { dc->dc_func = _dispatch_block_async_invoke_and_release; } else { dc->dc_func = _dispatch_block_async_invoke; @@ -3156,28 +3281,7 @@ _dispatch_continuation_init_slow(dispatch_continuation_t dc, dc->dc_flags = dc_flags; } -void -_dispatch_continuation_update_bits(dispatch_continuation_t dc, - uintptr_t dc_flags) -{ - dc->dc_flags = dc_flags; - if (dc_flags & DISPATCH_OBJ_CONSUME_BIT) { - if (dc_flags & DISPATCH_OBJ_BLOCK_PRIVATE_DATA_BIT) { - dc->dc_func = _dispatch_block_async_invoke_and_release; - } else if (dc_flags & DISPATCH_OBJ_BLOCK_BIT) { - dc->dc_func = _dispatch_call_block_and_release; - } - } else { - if (dc_flags & DISPATCH_OBJ_BLOCK_PRIVATE_DATA_BIT) { - dc->dc_func = _dispatch_block_async_invoke; - } else if (dc_flags & DISPATCH_OBJ_BLOCK_BIT) { - dc->dc_func = _dispatch_Block_invoke(dc->dc_ctxt); - } - } -} - #endif // __BLOCKS__ - #pragma mark - #pragma mark dispatch_barrier_async @@ -4092,15 +4196,19 @@ _dispatch_sync_block_with_private_data(dispatch_queue_t dq, dispatch_block_t work, dispatch_block_flags_t flags) { dispatch_block_private_data_t dbpd = _dispatch_block_get_data(work); - pthread_priority_t op = 0; + pthread_priority_t op = 0, p = 0; flags |= dbpd->dbpd_flags; - if (_dispatch_block_invoke_should_set_priority(flags)) { - voucher_t v = DISPATCH_NO_VOUCHER; - op = _dispatch_get_priority(); - v = _dispatch_set_priority_and_voucher(dbpd->dbpd_priority, v, 0); - dispatch_assert(v == DISPATCH_NO_VOUCHER); + op = _dispatch_block_invoke_should_set_priority(flags, dbpd->dbpd_priority); + if (op) { + p = dbpd->dbpd_priority; + } + voucher_t ov, v = DISPATCH_NO_VOUCHER; + if (flags & DISPATCH_BLOCK_HAS_VOUCHER) { + v = dbpd->dbpd_voucher; } + ov = _dispatch_set_priority_and_voucher(p, v, 0); + // balanced in d_block_sync_invoke or d_block_wait if (os_atomic_cmpxchg2o(dbpd, dbpd_queue, NULL, dq->_as_oq, relaxed)) { _dispatch_retain_2(dq); @@ -4110,7 +4218,7 @@ _dispatch_sync_block_with_private_data(dispatch_queue_t dq, } else { dispatch_sync_f(dq, work, _dispatch_block_sync_invoke); } - _dispatch_reset_priority_and_voucher(op, DISPATCH_NO_VOUCHER); + _dispatch_reset_priority_and_voucher(op, ov); } void @@ -4448,7 +4556,7 @@ _dispatch_global_queue_poke_slow(dispatch_queue_t dq, int n, int floor) #if HAVE_PTHREAD_WORKQUEUE_QOS r = _pthread_workqueue_addthreads(remaining, _dispatch_priority_to_pp(dq->dq_priority)); -#elif HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#elif DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP r = pthread_workqueue_addthreads_np(qc->dgq_wq_priority, qc->dgq_wq_options, remaining); #endif @@ -5250,7 +5358,7 @@ _dispatch_queue_class_wakeup_with_override_slow(dispatch_queue_t dq, } apply_again: - if (dx_type(tq) == DISPATCH_QUEUE_GLOBAL_ROOT_TYPE) { + if (dx_hastypeflag(tq, QUEUE_ROOT)) { if (_dispatch_root_queue_push_queue_override_needed(tq, qos)) { _dispatch_root_queue_push_override_stealer(tq, dq, qos); } @@ -5675,6 +5783,9 @@ _dispatch_root_queue_drain_deferred_wlh(dispatch_deferred_items_t ddi goto park; } dq_state = os_atomic_load2o(dq, dq_state, relaxed); + if (unlikely(!_dq_state_is_base_wlh(dq_state))) { // rdar://32671286 + goto park; + } if (unlikely(_dq_state_is_enqueued_on_target(dq_state))) { _dispatch_retain(dq); _dispatch_trace_continuation_push(dq->do_targetq, dq); @@ -5810,7 +5921,7 @@ _dispatch_worker_thread3(pthread_priority_t pp) } #endif // HAVE_PTHREAD_WORKQUEUE_QOS -#if HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#if DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP // 6618342 Contact the team that owns the Instrument DTrace probe before // renaming this symbol static void @@ -5823,7 +5934,7 @@ _dispatch_worker_thread2(int priority, int options, return _dispatch_worker_thread4(dq); } -#endif // HAVE_PTHREAD_WORKQUEUE_SETDISPATCH_NP +#endif // DISPATCH_USE_PTHREAD_WORKQUEUE_SETDISPATCH_NP #endif // HAVE_PTHREAD_WORKQUEUES #if DISPATCH_USE_PTHREAD_POOL diff --git a/src/queue_internal.h b/src/queue_internal.h index c1d0f6e5a..1a590e27a 100644 --- a/src/queue_internal.h +++ b/src/queue_internal.h @@ -832,6 +832,8 @@ dispatch_queue_attr_t _dispatch_get_default_queue_attr(void); #define DISPATCH_OBJ_CTXT_FETCH_BIT 0x040ul // use the voucher from the continuation even if the queue has voucher set #define DISPATCH_OBJ_ENFORCE_VOUCHER 0x080ul +// never set on continuations, used by mach.c only +#define DISPATCH_OBJ_MACH_BARRIER 0x1000000ul typedef struct dispatch_continuation_s { struct dispatch_object_s _as_do[0]; @@ -975,7 +977,7 @@ typedef struct dispatch_apply_s *dispatch_apply_t; #ifdef __BLOCKS__ -#define DISPATCH_BLOCK_API_MASK (0x80u - 1) +#define DISPATCH_BLOCK_API_MASK (0x100u - 1) #define DISPATCH_BLOCK_HAS_VOUCHER (1u << 31) #define DISPATCH_BLOCK_HAS_PRIORITY (1u << 30) @@ -1022,8 +1024,6 @@ void _dispatch_block_sync_invoke(void *block); void _dispatch_continuation_init_slow(dispatch_continuation_t dc, dispatch_queue_class_t dqu, dispatch_block_flags_t flags); -void _dispatch_continuation_update_bits(dispatch_continuation_t dc, - uintptr_t dc_flags); long _dispatch_barrier_trysync_f(dispatch_queue_t dq, void *ctxt, dispatch_function_t func); diff --git a/src/shims.h b/src/shims.h index 8dd23ee24..28e1c53a9 100644 --- a/src/shims.h +++ b/src/shims.h @@ -36,6 +36,7 @@ #include "shims/android_stubs.h" #endif +#include "shims/hw_config.h" #include "shims/priority.h" #if HAVE_PTHREAD_WORKQUEUES @@ -147,6 +148,51 @@ _pthread_workqueue_should_narrow(pthread_priority_t priority) } #endif +#if HAVE_PTHREAD_QOS_H && __has_include() && \ + defined(PTHREAD_MAX_PARALLELISM_PHYSICAL) && \ + DISPATCH_HAVE_HW_CONFIG_COMMPAGE && \ + DISPATCH_MIN_REQUIRED_OSX_AT_LEAST(109900) +#define DISPATCH_USE_PTHREAD_QOS_MAX_PARALLELISM 1 +#define DISPATCH_MAX_PARALLELISM_PHYSICAL PTHREAD_MAX_PARALLELISM_PHYSICAL +#else +#define DISPATCH_MAX_PARALLELISM_PHYSICAL 0x1 +#endif +#define DISPATCH_MAX_PARALLELISM_ACTIVE 0x2 +_Static_assert(!(DISPATCH_MAX_PARALLELISM_PHYSICAL & + DISPATCH_MAX_PARALLELISM_ACTIVE), "Overlapping parallelism flags"); + +DISPATCH_ALWAYS_INLINE +static inline uint32_t +_dispatch_qos_max_parallelism(dispatch_qos_t qos, unsigned long flags) +{ + uint32_t p; + int r = 0; + + if (qos) { +#if DISPATCH_USE_PTHREAD_QOS_MAX_PARALLELISM + r = pthread_qos_max_parallelism(_dispatch_qos_to_qos_class(qos), + flags & PTHREAD_MAX_PARALLELISM_PHYSICAL); +#endif + } + if (likely(r > 0)) { + p = (uint32_t)r; + } else { + p = (flags & DISPATCH_MAX_PARALLELISM_PHYSICAL) ? + dispatch_hw_config(physical_cpus) : + dispatch_hw_config(logical_cpus); + } + if (flags & DISPATCH_MAX_PARALLELISM_ACTIVE) { + uint32_t active_cpus = dispatch_hw_config(active_cpus); + if ((flags & DISPATCH_MAX_PARALLELISM_PHYSICAL) && + active_cpus < dispatch_hw_config(logical_cpus)) { + active_cpus /= dispatch_hw_config(logical_cpus) / + dispatch_hw_config(physical_cpus); + } + if (active_cpus < p) p = active_cpus; + } + return p; +} + #if !HAVE_NORETURN_BUILTIN_TRAP /* * XXXRW: Work-around for possible clang bug in which __builtin_trap() is not @@ -174,7 +220,6 @@ void __builtin_trap(void); #include "shims/yield.h" #include "shims/lock.h" -#include "shims/hw_config.h" #include "shims/perfmon.h" #include "shims/getprogname.h" @@ -228,7 +273,8 @@ _dispatch_mempcpy(void *ptr, const void *data, size_t len) #define _dispatch_clear_stack(s) do { \ void *a[(s)/sizeof(void*) ? (s)/sizeof(void*) : 1]; \ a[0] = pthread_get_stackaddr_np(pthread_self()); \ - bzero((void*)&a[1], (size_t)(a[0] - (void*)&a[1])); \ + void* volatile const p = (void*)&a[1]; /* */ \ + bzero((void*)p, (size_t)(a[0] - (void*)&a[1])); \ } while (0) #else #define _dispatch_clear_stack(s) diff --git a/src/shims/priority.h b/src/shims/priority.h index 948e4c7af..3e85ff54c 100644 --- a/src/shims/priority.h +++ b/src/shims/priority.h @@ -94,7 +94,7 @@ typedef uint16_t dispatch_priority_requested_t; #define DISPATCH_PRIORITY_OVERRIDE_SHIFT 16 #define DISPATCH_PRIORITY_FLAGS_MASK ((dispatch_priority_t)0xff000000) -#define DISPATCH_PRIORITY_SATURATED_OVERRIDE DISPATCH_PRIORITY_OVERRIDE_MASK +#define DISPATCH_PRIORITY_SATURATED_OVERRIDE ((dispatch_priority_t)0x000f0000) #define DISPATCH_PRIORITY_FLAG_OVERCOMMIT ((dispatch_priority_t)0x80000000) // _PTHREAD_PRIORITY_OVERCOMMIT_FLAG #define DISPATCH_PRIORITY_FLAG_DEFAULTQUEUE ((dispatch_priority_t)0x04000000) // _PTHREAD_PRIORITY_DEFAULTQUEUE_FLAG diff --git a/src/source.c b/src/source.c index fd337a9a3..6b9750722 100644 --- a/src/source.c +++ b/src/source.c @@ -521,6 +521,22 @@ _dispatch_source_latch_and_call(dispatch_source_t ds, dispatch_queue_t cq, } } +DISPATCH_NOINLINE +static void +_dispatch_source_refs_finalize_unregistration(dispatch_source_t ds) +{ + dispatch_queue_flags_t dqf; + dispatch_source_refs_t dr = ds->ds_refs; + + dqf = _dispatch_queue_atomic_flags_set_and_clear_orig(ds->_as_dq, + DSF_DELETED, DSF_ARMED | DSF_DEFERRED_DELETE | DSF_CANCEL_WAITER); + if (dqf & DSF_CANCEL_WAITER) { + _dispatch_wake_by_address(&ds->dq_atomic_flags); + } + _dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", ds, dr); + _dispatch_release_tailcall(ds); // the retain is done at creation time +} + void _dispatch_source_refs_unregister(dispatch_source_t ds, uint32_t options) { @@ -549,14 +565,8 @@ _dispatch_source_refs_unregister(dispatch_source_t ds, uint32_t options) } } - dqf = _dispatch_queue_atomic_flags_set_and_clear_orig(ds->_as_dq, - DSF_DELETED, DSF_ARMED | DSF_DEFERRED_DELETE | DSF_CANCEL_WAITER); - if (dqf & DSF_CANCEL_WAITER) { - _dispatch_wake_by_address(&ds->dq_atomic_flags); - } ds->ds_is_installed = true; - _dispatch_debug("kevent-source[%p]: disarmed kevent[%p]", ds, dr); - _dispatch_release_tailcall(ds); // the retain is done at creation time + _dispatch_source_refs_finalize_unregistration(ds); } DISPATCH_ALWAYS_INLINE @@ -619,8 +629,9 @@ _dispatch_source_refs_register(dispatch_source_t ds, dispatch_wlh_t wlh, if (unlikely(!_dispatch_source_tryarm(ds) || !_dispatch_unote_register(dr, wlh, pri))) { - _dispatch_queue_atomic_flags_set_and_clear(ds->_as_dq, DSF_DELETED, - DSF_ARMED | DSF_DEFERRED_DELETE); + // Do the parts of dispatch_source_refs_unregister() that + // are required after this partial initialization. + _dispatch_source_refs_finalize_unregistration(ds); } else { _dispatch_debug("kevent-source[%p]: armed kevent[%p]", ds, dr); } @@ -1761,8 +1772,10 @@ _dispatch_timer_heap_insert(dispatch_timer_heap_t dth, { uint32_t idx = (dth->dth_count += DTH_ID_COUNT) - DTH_ID_COUNT; - dispatch_assert(dt->dt_heap_entry[DTH_TARGET_ID] == DTH_INVALID_ID); - dispatch_assert(dt->dt_heap_entry[DTH_DEADLINE_ID] == DTH_INVALID_ID); + DISPATCH_TIMER_ASSERT(dt->dt_heap_entry[DTH_TARGET_ID], ==, + DTH_INVALID_ID, "target idx"); + DISPATCH_TIMER_ASSERT(dt->dt_heap_entry[DTH_DEADLINE_ID], ==, + DTH_INVALID_ID, "deadline idx"); if (idx == 0) { dt->dt_heap_entry[DTH_TARGET_ID] = DTH_TARGET_ID; @@ -1786,12 +1799,16 @@ _dispatch_timer_heap_remove(dispatch_timer_heap_t dth, { uint32_t idx = (dth->dth_count -= DTH_ID_COUNT); - dispatch_assert(dt->dt_heap_entry[DTH_TARGET_ID] != DTH_INVALID_ID); - dispatch_assert(dt->dt_heap_entry[DTH_DEADLINE_ID] != DTH_INVALID_ID); + DISPATCH_TIMER_ASSERT(dt->dt_heap_entry[DTH_TARGET_ID], !=, + DTH_INVALID_ID, "target idx"); + DISPATCH_TIMER_ASSERT(dt->dt_heap_entry[DTH_DEADLINE_ID], !=, + DTH_INVALID_ID, "deadline idx"); if (idx == 0) { - dispatch_assert(dth->dth_min[DTH_TARGET_ID] == dt); - dispatch_assert(dth->dth_min[DTH_DEADLINE_ID] == dt); + DISPATCH_TIMER_ASSERT(dth->dth_min[DTH_TARGET_ID], ==, dt, + "target slot"); + DISPATCH_TIMER_ASSERT(dth->dth_min[DTH_DEADLINE_ID], ==, dt, + "deadline slot"); dth->dth_min[DTH_TARGET_ID] = dth->dth_min[DTH_DEADLINE_ID] = NULL; goto clear_heap_entry; } @@ -1819,8 +1836,11 @@ static inline void _dispatch_timer_heap_update(dispatch_timer_heap_t dth, dispatch_timer_source_refs_t dt) { - dispatch_assert(dt->dt_heap_entry[DTH_TARGET_ID] != DTH_INVALID_ID); - dispatch_assert(dt->dt_heap_entry[DTH_DEADLINE_ID] != DTH_INVALID_ID); + DISPATCH_TIMER_ASSERT(dt->dt_heap_entry[DTH_TARGET_ID], !=, + DTH_INVALID_ID, "target idx"); + DISPATCH_TIMER_ASSERT(dt->dt_heap_entry[DTH_DEADLINE_ID], !=, + DTH_INVALID_ID, "deadline idx"); + _dispatch_timer_heap_resift(dth, dt, dt->dt_heap_entry[DTH_TARGET_ID]); _dispatch_timer_heap_resift(dth, dt, dt->dt_heap_entry[DTH_DEADLINE_ID]); @@ -1875,7 +1895,7 @@ _dispatch_timers_register(dispatch_timer_source_refs_t dt, uint32_t tidx) { dispatch_timer_heap_t heap = &_dispatch_timers_heap[tidx]; if (_dispatch_unote_registered(dt)) { - dispatch_assert(dt->du_ident == tidx); + DISPATCH_TIMER_ASSERT(dt->du_ident, ==, tidx, "tidx"); _dispatch_timer_heap_update(heap, dt); } else { dt->du_ident = tidx; @@ -2051,7 +2071,10 @@ _dispatch_timers_run2(dispatch_clock_now_cache_t nows, uint32_t tidx) uint64_t now = _dispatch_time_now_cached(DISPATCH_TIMER_CLOCK(tidx), nows); while ((dr = _dispatch_timers_heap[tidx].dth_min[DTH_TARGET_ID])) { - dispatch_assert(tidx == dr->du_ident && dr->dt_timer.target); + DISPATCH_TIMER_ASSERT(dr->du_filter, ==, DISPATCH_EVFILT_TIMER, + "invalid filter"); + DISPATCH_TIMER_ASSERT(dr->du_ident, ==, tidx, "tidx"); + DISPATCH_TIMER_ASSERT(dr->dt_timer.target, !=, 0, "missing target"); ds = _dispatch_source_from_refs(dr); if (dr->dt_timer.target > now) { // Done running timers for now. diff --git a/src/swift/Source.swift b/src/swift/Source.swift index a3a7e7903..421a6e9bb 100644 --- a/src/swift/Source.swift +++ b/src/swift/Source.swift @@ -279,28 +279,342 @@ public extension DispatchSourceProcess { #endif public extension DispatchSourceTimer { + /// + /// Sets the deadline and leeway for a timer event that fires once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared and the next timer event will occur at `deadline`. + /// + /// Delivery of the timer event may be delayed by the system in order to improve power consumption + /// and system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// - note: Delivery of the timer event does not cancel the timer source. + /// + /// - parameter deadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on Mach absolute + /// time. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, deprecated: 4, renamed: "schedule(deadline:repeating:leeway:)") public func scheduleOneshot(deadline: DispatchTime, leeway: DispatchTimeInterval = .nanoseconds(0)) { dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, ~0, UInt64(leeway.rawValue)) } + /// + /// Sets the deadline and leeway for a timer event that fires once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared and the next timer event will occur at `wallDeadline`. + /// + /// Delivery of the timer event may be delayed by the system in order to improve power consumption + /// and system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// - note: Delivery of the timer event does not cancel the timer source. + /// + /// - parameter wallDeadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on + /// `gettimeofday(3)`. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, deprecated: 4, renamed: "schedule(wallDeadline:repeating:leeway:)") public func scheduleOneshot(wallDeadline: DispatchWallTime, leeway: DispatchTimeInterval = .nanoseconds(0)) { dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, ~0, UInt64(leeway.rawValue)) } + /// + /// Sets the deadline, interval and leeway for a timer event that fires at least once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `deadline` and every `interval` units of + /// time thereafter until the timer source is canceled. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption + /// and system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `deadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `deadline + N * interval`, the upper + /// limit is the smaller of `leeway` and `interval/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter deadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on Mach absolute + /// time. + /// - parameter interval: the interval for the timer. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, deprecated: 4, renamed: "schedule(deadline:repeating:leeway:)") public func scheduleRepeating(deadline: DispatchTime, interval: DispatchTimeInterval, leeway: DispatchTimeInterval = .nanoseconds(0)) { - dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, UInt64(interval.rawValue), UInt64(leeway.rawValue)) + dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, interval == .never ? ~0 : UInt64(interval.rawValue), UInt64(leeway.rawValue)) } + /// + /// Sets the deadline, interval and leeway for a timer event that fires at least once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `deadline` and every `interval` seconds + /// thereafter until the timer source is canceled. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption and + /// system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `deadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `deadline + N * interval`, the upper + /// limit is the smaller of `leeway` and `interval/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter deadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on Mach absolute + /// time. + /// - parameter interval: the interval for the timer in seconds. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, deprecated: 4, renamed: "schedule(deadline:repeating:leeway:)") public func scheduleRepeating(deadline: DispatchTime, interval: Double, leeway: DispatchTimeInterval = .nanoseconds(0)) { - dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, UInt64(interval * Double(NSEC_PER_SEC)), UInt64(leeway.rawValue)) + dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, interval.isInfinite ? ~0 : UInt64(interval * Double(NSEC_PER_SEC)), UInt64(leeway.rawValue)) } + /// + /// Sets the deadline, interval and leeway for a timer event that fires at least once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `wallDeadline` and every `interval` units of + /// time thereafter until the timer source is canceled. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption and + /// system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `wallDeadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `wallDeadline + N * interval`, the upper + /// limit is the smaller of `leeway` and `interval/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter wallDeadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on + /// `gettimeofday(3)`. + /// - parameter interval: the interval for the timer. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, deprecated: 4, renamed: "schedule(wallDeadline:repeating:leeway:)") public func scheduleRepeating(wallDeadline: DispatchWallTime, interval: DispatchTimeInterval, leeway: DispatchTimeInterval = .nanoseconds(0)) { - dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, UInt64(interval.rawValue), UInt64(leeway.rawValue)) + dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, interval == .never ? ~0 : UInt64(interval.rawValue), UInt64(leeway.rawValue)) } + /// + /// Sets the deadline, interval and leeway for a timer event that fires at least once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `wallDeadline` and every `interval` seconds + /// thereafter until the timer source is canceled. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption and + /// system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `wallDeadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `wallDeadline + N * interval`, the upper + /// limit is the smaller of `leeway` and `interval/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter wallDeadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on + /// `gettimeofday(3)`. + /// - parameter interval: the interval for the timer in seconds. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, deprecated: 4, renamed: "schedule(wallDeadline:repeating:leeway:)") public func scheduleRepeating(wallDeadline: DispatchWallTime, interval: Double, leeway: DispatchTimeInterval = .nanoseconds(0)) { - dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, UInt64(interval * Double(NSEC_PER_SEC)), UInt64(leeway.rawValue)) + dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, interval.isInfinite ? ~0 : UInt64(interval * Double(NSEC_PER_SEC)), UInt64(leeway.rawValue)) + } + + /// + /// Sets the deadline, repeat interval and leeway for a timer event. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `deadline` and every `repeating` units of + /// time thereafter until the timer source is canceled. If the value of `repeating` is `.never`, + /// or is defaulted, the timer fires only once. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption + /// and system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `deadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `deadline + N * repeating`, the upper + /// limit is the smaller of `leeway` and `repeating/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter deadline: the time at which the first timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on Mach absolute + /// time. + /// - parameter repeating: the repeat interval for the timer, or `.never` if the timer should fire + /// only once. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, introduced: 4) + public func schedule(deadline: DispatchTime, repeating interval: DispatchTimeInterval = .never, leeway: DispatchTimeInterval = .nanoseconds(0)) { + dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, interval == .never ? ~0 : UInt64(interval.rawValue), UInt64(leeway.rawValue)) + } + + /// + /// Sets the deadline, repeat interval and leeway for a timer event. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `deadline` and every `repeating` seconds + /// thereafter until the timer source is canceled. If the value of `repeating` is `.infinity`, + /// the timer fires only once. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption + /// and system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `deadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `deadline + N * repeating`, the upper + /// limit is the smaller of `leeway` and `repeating/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter deadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on Mach absolute + /// time. + /// - parameter repeating: the repeat interval for the timer in seconds, or `.infinity` if the timer + /// should fire only once. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, introduced: 4) + public func schedule(deadline: DispatchTime, repeating interval: Double, leeway: DispatchTimeInterval = .nanoseconds(0)) { + dispatch_source_set_timer((self as! DispatchSource).__wrapped, deadline.rawValue, interval.isInfinite ? ~0 : UInt64(interval * Double(NSEC_PER_SEC)), UInt64(leeway.rawValue)) + } + + /// + /// Sets the deadline, repeat interval and leeway for a timer event. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `wallDeadline` and every `repeating` units of + /// time thereafter until the timer source is canceled. If the value of `repeating` is `.never`, + /// or is defaulted, the timer fires only once. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption and + /// system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `wallDeadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `wallDeadline + N * repeating`, the upper + /// limit is the smaller of `leeway` and `repeating/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter wallDeadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on + /// `gettimeofday(3)`. + /// - parameter repeating: the repeat interval for the timer, or `.never` if the timer should fire + /// only once. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, introduced: 4) + public func schedule(wallDeadline: DispatchWallTime, repeating interval: DispatchTimeInterval = .never, leeway: DispatchTimeInterval = .nanoseconds(0)) { + dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, interval == .never ? ~0 : UInt64(interval.rawValue), UInt64(leeway.rawValue)) + } + + /// + /// Sets the deadline, repeat interval and leeway for a timer event that fires at least once. + /// + /// Once this function returns, any pending source data accumulated for the previous timer values + /// has been cleared. The next timer event will occur at `wallDeadline` and every `repeating` seconds + /// thereafter until the timer source is canceled. If the value of `repeating` is `.infinity`, + /// the timer fires only once. + /// + /// Delivery of a timer event may be delayed by the system in order to improve power consumption + /// and system performance. The upper limit to the allowable delay may be configured with the `leeway` + /// argument; the lower limit is under the control of the system. + /// + /// For the initial timer fire at `wallDeadline`, the upper limit to the allowable delay is set to + /// `leeway`. For the subsequent timer fires at `wallDeadline + N * repeating`, the upper + /// limit is the smaller of `leeway` and `repeating/2`. + /// + /// The lower limit to the allowable delay may vary with process state such as visibility of the + /// application UI. If the timer source was created with flags `TimerFlags.strict`, the system + /// will make a best effort to strictly observe the provided `leeway` value, even if it is smaller + /// than the current lower limit. Note that a minimal amount of delay is to be expected even if + /// this flag is specified. + /// + /// Calling this method has no effect if the timer source has already been canceled. + /// + /// - parameter wallDeadline: the time at which the timer event will be delivered, subject to the + /// leeway and other considerations described above. The deadline is based on + /// `gettimeofday(3)`. + /// - parameter repeating: the repeat interval for the timer in seconds, or `.infinity` if the timer + /// should fire only once. + /// - parameter leeway: the leeway for the timer. + /// + @available(swift, introduced: 4) + public func schedule(wallDeadline: DispatchWallTime, repeating interval: Double, leeway: DispatchTimeInterval = .nanoseconds(0)) { + dispatch_source_set_timer((self as! DispatchSource).__wrapped, wallDeadline.rawValue, interval.isInfinite ? ~0 : UInt64(interval * Double(NSEC_PER_SEC)), UInt64(leeway.rawValue)) } } diff --git a/src/swift/Time.swift b/src/swift/Time.swift index 8178ffd6c..d7d49c96b 100644 --- a/src/swift/Time.swift +++ b/src/swift/Time.swift @@ -124,6 +124,8 @@ public enum DispatchTimeInterval { case milliseconds(Int) case microseconds(Int) case nanoseconds(Int) + @_downgrade_exhaustivity_check + case never internal var rawValue: Int64 { switch self { @@ -131,6 +133,16 @@ public enum DispatchTimeInterval { case .milliseconds(let ms): return Int64(ms) * Int64(NSEC_PER_MSEC) case .microseconds(let us): return Int64(us) * Int64(NSEC_PER_USEC) case .nanoseconds(let ns): return Int64(ns) + case .never: return Int64.max + } + } + + public static func ==(lhs: DispatchTimeInterval, rhs: DispatchTimeInterval) -> Bool { + switch (lhs, rhs) { + case (.never, .never): return true + case (.never, _): return false + case (_, .never): return false + default: return lhs.rawValue == rhs.rawValue } } } diff --git a/src/trace.h b/src/trace.h index 872cd6ff5..c670f60b7 100644 --- a/src/trace.h +++ b/src/trace.h @@ -29,14 +29,6 @@ #if DISPATCH_PURE_C -#if DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION -typedef struct dispatch_trace_timer_params_s { - int64_t deadline, interval, leeway; -} *dispatch_trace_timer_params_t; - -#include "provider.h" -#endif // DISPATCH_USE_DTRACE || DISPATCH_USE_DTRACE_INTROSPECTION - #if DISPATCH_USE_DTRACE_INTROSPECTION #define _dispatch_trace_callout(_c, _f, _dcc) do { \ if (slowpath(DISPATCH_CALLOUT_ENTRY_ENABLED()) || \ diff --git a/src/voucher.c b/src/voucher.c index e4128a289..458e2f0a4 100644 --- a/src/voucher.c +++ b/src/voucher.c @@ -85,6 +85,7 @@ voucher_create(voucher_recipe_t recipe) if (extra) { memcpy(_voucher_extra_recipes(voucher), recipe->vr_data, extra); } + _voucher_trace(CREATE, voucher, MACH_PORT_NULL, 0); return voucher; } #endif @@ -585,6 +586,7 @@ _voucher_create_with_mach_voucher(mach_voucher_t kv, mach_msg_bits_t msgh_bits) } } + _voucher_trace(CREATE, v, v->v_kvoucher, v->v_activity); _voucher_insert(v); _dispatch_voucher_debug("kvoucher[0x%08x] create", v, kv); return v; @@ -619,6 +621,7 @@ _voucher_create_with_priority_and_mach_voucher(voucher_t ov, "voucher[%p]", v, kv, ov); _dispatch_voucher_debug_machport(kv); } + _voucher_trace(CREATE, v, v->v_kvoucher, v->v_activity); return v; } @@ -676,6 +679,7 @@ _voucher_create_without_importance(voucher_t ov) _dispatch_voucher_debug("kvoucher[0x%08x] create without importance " "from voucher[%p]", v, kv, ov); } + _voucher_trace(CREATE, v, v->v_kvoucher, v->v_activity); return v; } @@ -711,6 +715,7 @@ _voucher_create_accounting_voucher(voucher_t ov) v->v_kvbase = _voucher_retain(ov); _voucher_dealloc_mach_voucher(kv); // borrow base reference } + _voucher_trace(CREATE, v, kv, v->v_activity); _voucher_insert(v); _dispatch_voucher_debug("kvoucher[0x%08x] create accounting voucher " "from voucher[%p]", v, kv, ov); @@ -774,6 +779,7 @@ _voucher_xref_dispose(voucher_t voucher) void _voucher_dispose(voucher_t voucher) { + _voucher_trace(DISPOSE, voucher); _dispatch_voucher_debug("dispose", voucher); if (slowpath(_voucher_hash_is_enqueued(voucher))) { _dispatch_voucher_debug("corruption", voucher); @@ -1237,6 +1243,7 @@ voucher_activity_create_with_data(firehose_tracepoint_id_t *trace_id, } done: *trace_id = ftid.ftid_value; + _voucher_trace(CREATE, v, v->v_kvoucher, va_id); return v; } diff --git a/src/voucher_internal.h b/src/voucher_internal.h index a0ddd4db4..772c8c434 100644 --- a/src/voucher_internal.h +++ b/src/voucher_internal.h @@ -262,6 +262,16 @@ typedef struct voucher_recipe_s { #define _dispatch_voucher_debug_machport(name) ((void)(name)) #endif +#if DISPATCH_USE_DTRACE +#define _voucher_trace(how, ...) ({ \ + if (unlikely(VOUCHER_##how##_ENABLED())) { \ + VOUCHER_##how(__VA_ARGS__); \ + } \ + }) +#else +#define _voucher_trace(how, ...) ((void)0) +#endif + #ifndef DISPATCH_VOUCHER_OBJC_DEBUG #if DISPATCH_INTROSPECTION || DISPATCH_DEBUG #define DISPATCH_VOUCHER_OBJC_DEBUG 1 @@ -270,36 +280,29 @@ typedef struct voucher_recipe_s { #endif #endif // DISPATCH_VOUCHER_OBJC_DEBUG -#if DISPATCH_PURE_C - DISPATCH_ALWAYS_INLINE -static inline voucher_t -_voucher_retain(voucher_t voucher) +static inline struct voucher_s * +_voucher_retain_inline(struct voucher_s *voucher) { -#if !DISPATCH_VOUCHER_OBJC_DEBUG // not using _os_object_refcnt* because we don't need barriers: // vouchers are immutable and are in a hash table with a lock int xref_cnt = os_atomic_inc2o(voucher, os_obj_xref_cnt, relaxed); + _voucher_trace(RETAIN, (voucher_t)voucher, xref_cnt + 1); _dispatch_voucher_debug("retain -> %d", voucher, xref_cnt + 1); if (unlikely(xref_cnt <= 0)) { _OS_OBJECT_CLIENT_CRASH("Voucher resurrection"); } -#else - os_retain(voucher); - _dispatch_voucher_debug("retain -> %d", voucher, - voucher->os_obj_xref_cnt + 1); -#endif // DISPATCH_DEBUG return voucher; } DISPATCH_ALWAYS_INLINE static inline void -_voucher_release(voucher_t voucher) +_voucher_release_inline(struct voucher_s *voucher) { -#if !DISPATCH_VOUCHER_OBJC_DEBUG // not using _os_object_refcnt* because we don't need barriers: // vouchers are immutable and are in a hash table with a lock int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); + _voucher_trace(RELEASE, (voucher_t)voucher, xref_cnt + 1); _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); if (likely(xref_cnt >= 0)) { return; @@ -308,10 +311,31 @@ _voucher_release(voucher_t voucher) _OS_OBJECT_CLIENT_CRASH("Voucher over-release"); } return _os_object_xref_dispose((_os_object_t)voucher); +} + +#if DISPATCH_PURE_C + +DISPATCH_ALWAYS_INLINE +static inline voucher_t +_voucher_retain(voucher_t voucher) +{ +#if DISPATCH_VOUCHER_OBJC_DEBUG + os_retain(voucher); #else - _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt); - return os_release(voucher); -#endif // DISPATCH_DEBUG + _voucher_retain_inline(voucher); +#endif // DISPATCH_VOUCHER_OBJC_DEBUG + return voucher; +} + +DISPATCH_ALWAYS_INLINE +static inline void +_voucher_release(voucher_t voucher) +{ +#if DISPATCH_VOUCHER_OBJC_DEBUG + os_release(voucher); +#else + _voucher_release_inline(voucher); +#endif // DISPATCH_VOUCHER_OBJC_DEBUG } DISPATCH_ALWAYS_INLINE @@ -322,13 +346,13 @@ _voucher_release_no_dispose(voucher_t voucher) // not using _os_object_refcnt* because we don't need barriers: // vouchers are immutable and are in a hash table with a lock int xref_cnt = os_atomic_dec2o(voucher, os_obj_xref_cnt, relaxed); + _voucher_trace(RELEASE, voucher, xref_cnt + 1); _dispatch_voucher_debug("release -> %d", voucher, xref_cnt + 1); if (likely(xref_cnt >= 0)) { return; } _OS_OBJECT_CLIENT_CRASH("Voucher over-release"); #else - _dispatch_voucher_debug("release -> %d", voucher, voucher->os_obj_xref_cnt); return os_release(voucher); #endif // DISPATCH_DEBUG } @@ -371,8 +395,10 @@ static inline mach_voucher_t _voucher_swap_and_get_mach_voucher(voucher_t ov, voucher_t voucher) { if (ov == voucher) return VOUCHER_NO_MACH_VOUCHER; - _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov); + if (ov) _voucher_trace(ORPHAN, ov); _dispatch_thread_setspecific(dispatch_voucher_key, voucher); + if (voucher) _voucher_trace(ADOPT, voucher); + _dispatch_voucher_debug("swap from voucher[%p]", voucher, ov); mach_voucher_t kv = voucher ? voucher->v_kvoucher : MACH_VOUCHER_NULL; mach_voucher_t okv = ov ? ov->v_kvoucher : MACH_VOUCHER_NULL; #if OS_VOUCHER_ACTIVITY_GENERATE_SWAPS diff --git a/tools/voucher_trace.d b/tools/voucher_trace.d new file mode 100755 index 000000000..890198e66 --- /dev/null +++ b/tools/voucher_trace.d @@ -0,0 +1,78 @@ +#!/usr/sbin/dtrace -s + +/* + * Copyright (c) 2017 Apple Inc. All rights reserved. + * + * @APPLE_APACHE_LICENSE_HEADER_START@ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @APPLE_APACHE_LICENSE_HEADER_END@ + */ + +/* + * Usage: voucher_trace.d -p [pid] + * traced process must have been executed with + * DYLD_LIBRARY_PATH=/usr/lib/system/introspection or with + * DYLD_IMAGE_SUFFIX=_profile or DYLD_IMAGE_SUFFIX=_debug + */ + +#pragma D option quiet +#pragma D option zdefs +#pragma D option bufsize=16m + +BEGIN { + printf("Starting to trace voucher operations...\n"); +} + +voucher$target:libdispatch*.dylib::create +{ + printf("ALLOC voucher 0x%p, thread %#llx, ref 1, port %#x, aid %#llx", arg0, tid, arg1, arg2); + ustack(10); + printf("\n") +} + +voucher$target:libdispatch*.dylib::dispose +{ + printf("FREE voucher 0x%p, thread %#llx, ref 0", arg0, tid); + ustack(10); + printf("\n") +} + +voucher$target:libdispatch*.dylib::retain +{ + printf("RETAIN voucher 0x%p, thread %#llx, ref %d", arg0, tid, arg1); + ustack(10); + printf("\n") +} + +voucher$target:libdispatch*.dylib::release +{ + printf("RELEASE voucher 0x%p, thread %#llx, ref %d", arg0, tid, arg1); + ustack(10); + printf("\n") +} + +voucher$target:libdispatch*.dylib::adopt +{ + printf("ADOPT voucher 0x%p, thread %#llx", arg0, tid); + ustack(10); + printf("\n") +} + +voucher$target:libdispatch*.dylib::orphan +{ + printf("ORPHAN voucher 0x%p, thread %#llx", arg0, tid); + ustack(10); + printf("\n") +} diff --git a/xcodeconfig/libdispatch.order b/xcodeconfig/libdispatch.order index a25ecc980..9642ca4dd 100644 --- a/xcodeconfig/libdispatch.order +++ b/xcodeconfig/libdispatch.order @@ -71,6 +71,18 @@ _OBJC_METACLASS_$_OS_dispatch_queue _OBJC_METACLASS_$_OS_dispatch_queue_serial _OBJC_METACLASS_$_OS_dispatch_queue_concurrent _OBJC_METACLASS_$_OS_dispatch_queue_root +_OBJC_METACLASS_$_OS_dispatch_queue_main +_OBJC_METACLASS_$_OS_dispatch_queue_runloop +_OBJC_METACLASS_$_OS_dispatch_queue_mgr +_OBJC_METACLASS_$_OS_dispatch_queue_specific_queue +_OBJC_METACLASS_$_OS_dispatch_queue_attr +_OBJC_METACLASS_$_OS_dispatch_source +_OBJC_METACLASS_$_OS_dispatch_mach +_OBJC_METACLASS_$_OS_dispatch_mach_msg +_OBJC_METACLASS_$_OS_dispatch_io +_OBJC_METACLASS_$_OS_dispatch_operation +_OBJC_METACLASS_$_OS_dispatch_disk +_OBJC_METACLASS_$_OS_object _OBJC_METACLASS_$_OS_voucher #_OBJC_METACLASS_$_OS_voucher_recipe _OBJC_METACLASS_$_OS_dispatch_data