From e7f28d97ab84febbedf08e553bcd79ea7a773d57 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 11 Jun 2019 13:52:59 +0300 Subject: [PATCH 1/2] Eliminate RuntimeError raised by asyncio.all_tasks() If internal tasks weak set is changed by another thread during iteration. --- Lib/asyncio/tasks.py | 28 ++++++++++++++++--- .../2019-06-11-13-52-04.bpo-36607.5_mJkQ.rst | 2 ++ 2 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-06-11-13-52-04.bpo-36607.5_mJkQ.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 95e85600a2e802..276a1b166e6177 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -42,9 +42,19 @@ def all_tasks(loop=None): """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - # NB: set(_all_tasks) is required to protect + # NB: list(_all_tasks) is required to protect # from https://bugs.python.org/issue34970 bug - return {t for t in list(_all_tasks) + # NB: Have to repeat on RuntimeError, other thread + # can modify _all_tasks during list(_all_tasks) call + # https://bugs.python.org/issue36607 + while True: + try: + tasks = list(_all_tasks) + except RuntimeError: + pass + else: + break + return {t for t in tasks if futures._get_loop(t) is loop and not t.done()} @@ -54,9 +64,19 @@ def _all_tasks_compat(loop=None): # method. if loop is None: loop = events.get_event_loop() - # NB: set(_all_tasks) is required to protect + # NB: list(_all_tasks) is required to protect # from https://bugs.python.org/issue34970 bug - return {t for t in list(_all_tasks) if futures._get_loop(t) is loop} + # NB: Have to repeat on RuntimeError, other thread + # can modify _all_tasks during list(_all_tasks) call + # https://bugs.python.org/issue36607 + while True: + try: + tasks = list(_all_tasks) + except RuntimeError: + pass + else: + break + return {t for t in tasks if futures._get_loop(t) is loop} def _set_task_name(task, name): diff --git a/Misc/NEWS.d/next/Library/2019-06-11-13-52-04.bpo-36607.5_mJkQ.rst b/Misc/NEWS.d/next/Library/2019-06-11-13-52-04.bpo-36607.5_mJkQ.rst new file mode 100644 index 00000000000000..337c8e2668c49e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-06-11-13-52-04.bpo-36607.5_mJkQ.rst @@ -0,0 +1,2 @@ +Eliminate :exc:`RuntimeError` raised by :func:`asyncio.all_tasks()` if +internal tasks weak set is changed by another thread during iteration. From eb3d2a39684a4d6573812c4594f026db9bb3449a Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 11 Jun 2019 18:02:18 +0300 Subject: [PATCH 2/2] Address comments --- Lib/asyncio/tasks.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 276a1b166e6177..cd4832cfee246b 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -42,16 +42,19 @@ def all_tasks(loop=None): """Return a set of all tasks for the loop.""" if loop is None: loop = events.get_running_loop() - # NB: list(_all_tasks) is required to protect - # from https://bugs.python.org/issue34970 bug - # NB: Have to repeat on RuntimeError, other thread - # can modify _all_tasks during list(_all_tasks) call - # https://bugs.python.org/issue36607 + # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another + # thread while we do so. Therefore we cast it to list prior to filtering. The list + # cast itself requires iteration, so we repeat it several times ignoring + # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for + # details. + i = 0 while True: try: tasks = list(_all_tasks) except RuntimeError: - pass + i += 1 + if i >= 1000: + raise else: break return {t for t in tasks @@ -64,16 +67,19 @@ def _all_tasks_compat(loop=None): # method. if loop is None: loop = events.get_event_loop() - # NB: list(_all_tasks) is required to protect - # from https://bugs.python.org/issue34970 bug - # NB: Have to repeat on RuntimeError, other thread - # can modify _all_tasks during list(_all_tasks) call - # https://bugs.python.org/issue36607 + # Looping over a WeakSet (_all_tasks) isn't safe as it can be updated from another + # thread while we do so. Therefore we cast it to list prior to filtering. The list + # cast itself requires iteration, so we repeat it several times ignoring + # RuntimeErrors (which are not very likely to occur). See issues 34970 and 36607 for + # details. + i = 0 while True: try: tasks = list(_all_tasks) except RuntimeError: - pass + i += 1 + if i >= 1000: + raise else: break return {t for t in tasks if futures._get_loop(t) is loop}