68
68
class _ThreadWakeup :
69
69
def __init__ (self ):
70
70
self ._closed = False
71
+ self ._lock = threading .Lock ()
71
72
self ._reader , self ._writer = mp .Pipe (duplex = False )
72
73
73
74
def close (self ):
74
- # Please note that we do not take the shutdown lock when
75
+ # Please note that we do not take the self._lock when
75
76
# calling clear() (to avoid deadlocking) so this method can
76
77
# only be called safely from the same thread as all calls to
77
- # clear() even if you hold the shutdown lock. Otherwise we
78
+ # clear() even if you hold the lock. Otherwise we
78
79
# might try to read from the closed pipe.
79
- if not self ._closed :
80
- self ._closed = True
81
- self ._writer .close ()
82
- self ._reader .close ()
80
+ with self ._lock :
81
+ if not self ._closed :
82
+ self ._closed = True
83
+ self ._writer .close ()
84
+ self ._reader .close ()
83
85
84
86
def wakeup (self ):
85
- if not self ._closed :
86
- self ._writer .send_bytes (b"" )
87
+ with self ._lock :
88
+ if not self ._closed :
89
+ self ._writer .send_bytes (b"" )
87
90
88
91
def clear (self ):
89
- if not self ._closed :
90
- while self ._reader .poll ():
91
- self ._reader .recv_bytes ()
92
+ if self ._closed :
93
+ raise RuntimeError ('operation on closed _ThreadWakeup' )
94
+ while self ._reader .poll ():
95
+ self ._reader .recv_bytes ()
92
96
93
97
94
98
def _python_exit ():
@@ -167,10 +171,8 @@ def __init__(self, work_id, fn, args, kwargs):
167
171
168
172
class _SafeQueue (Queue ):
169
173
"""Safe Queue set exception to the future object linked to a job"""
170
- def __init__ (self , max_size = 0 , * , ctx , pending_work_items , shutdown_lock ,
171
- thread_wakeup ):
174
+ def __init__ (self , max_size = 0 , * , ctx , pending_work_items , thread_wakeup ):
172
175
self .pending_work_items = pending_work_items
173
- self .shutdown_lock = shutdown_lock
174
176
self .thread_wakeup = thread_wakeup
175
177
super ().__init__ (max_size , ctx = ctx )
176
178
@@ -179,8 +181,7 @@ def _on_queue_feeder_error(self, e, obj):
179
181
tb = format_exception (type (e ), e , e .__traceback__ )
180
182
e .__cause__ = _RemoteTraceback ('\n """\n {}"""' .format ('' .join (tb )))
181
183
work_item = self .pending_work_items .pop (obj .work_id , None )
182
- with self .shutdown_lock :
183
- self .thread_wakeup .wakeup ()
184
+ self .thread_wakeup .wakeup ()
184
185
# work_item can be None if another process terminated. In this
185
186
# case, the executor_manager_thread fails all work_items
186
187
# with BrokenProcessPool
@@ -305,12 +306,10 @@ def __init__(self, executor):
305
306
# will wake up the queue management thread so that it can terminate
306
307
# if there is no pending work item.
307
308
def weakref_cb (_ ,
308
- thread_wakeup = self .thread_wakeup ,
309
- shutdown_lock = self .shutdown_lock ):
309
+ thread_wakeup = self .thread_wakeup ):
310
310
mp .util .debug ('Executor collected: triggering callback for'
311
311
' QueueManager wakeup' )
312
- with shutdown_lock :
313
- thread_wakeup .wakeup ()
312
+ thread_wakeup .wakeup ()
314
313
315
314
self .executor_reference = weakref .ref (executor , weakref_cb )
316
315
@@ -438,11 +437,6 @@ def wait_result_broken_or_wakeup(self):
438
437
elif wakeup_reader in ready :
439
438
is_broken = False
440
439
441
- # No need to hold the _shutdown_lock here because:
442
- # 1. we're the only thread to use the wakeup reader
443
- # 2. we're also the only thread to call thread_wakeup.close()
444
- # 3. we want to avoid a possible deadlock when both reader and writer
445
- # would block (gh-105829)
446
440
self .thread_wakeup .clear ()
447
441
448
442
return result_item , is_broken , cause
@@ -740,10 +734,9 @@ def __init__(self, max_workers=None, mp_context=None,
740
734
# as it could result in a deadlock if a worker process dies with the
741
735
# _result_queue write lock still acquired.
742
736
#
743
- # _shutdown_lock must be locked to access _ThreadWakeup.close() and
744
- # .wakeup(). Care must also be taken to not call clear or close from
745
- # more than one thread since _ThreadWakeup.clear() is not protected by
746
- # the _shutdown_lock
737
+ # Care must be taken to only call clear and close from the
738
+ # executor_manager_thread, since _ThreadWakeup.clear() is not protected
739
+ # by a lock.
747
740
self ._executor_manager_thread_wakeup = _ThreadWakeup ()
748
741
749
742
# Create communication channels for the executor
@@ -754,7 +747,6 @@ def __init__(self, max_workers=None, mp_context=None,
754
747
self ._call_queue = _SafeQueue (
755
748
max_size = queue_size , ctx = self ._mp_context ,
756
749
pending_work_items = self ._pending_work_items ,
757
- shutdown_lock = self ._shutdown_lock ,
758
750
thread_wakeup = self ._executor_manager_thread_wakeup )
759
751
# Killed worker processes can produce spurious "broken pipe"
760
752
# tracebacks in the queue's own worker thread. But we detect killed
0 commit comments