From 8f58d77bc462a86048fd73dbe4f65d52028c426f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Jan 2025 15:41:10 +0100 Subject: [PATCH 01/11] gh-128679: Fix tracemalloc.stop() race condition Check again 'tracemalloc_config.tracing' once the GIL is held in tracemalloc_raw_alloc() and PyTraceMalloc_Track(), since another thread can call tracemalloc.stop() during PyGILState_Ensure() call. --- ...-01-10-15-43-52.gh-issue-128679.KcfVVR.rst | 3 +++ Python/tracemalloc.c | 25 ++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst diff --git a/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst b/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst new file mode 100644 index 00000000000000..837f90df07a705 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-01-10-15-43-52.gh-issue-128679.KcfVVR.rst @@ -0,0 +1,3 @@ +Fix :func:`tracemalloc.stop` race condition. Fix :mod:`tracemalloc` to +support calling :func:`tracemalloc.stop` in one thread, while another thread +is tracing memory allocations. Patch by Victor Stinner. diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index f661d69c0312fa..51f65c7e13857c 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -712,7 +712,18 @@ tracemalloc_raw_alloc(int use_calloc, void *ctx, size_t nelem, size_t elsize) set_reentrant(1); gil_state = PyGILState_Ensure(); - ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize); + if (tracemalloc_config.tracing) { + ptr = tracemalloc_alloc(use_calloc, ctx, nelem, elsize); + } + else { + // gh-128679: tracemalloc.stop() was called by another thread during + // PyGILState_Ensure() call. + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + if (use_calloc) + ptr = alloc->calloc(alloc->ctx, nelem, elsize); + else + ptr = alloc->malloc(alloc->ctx, nelem * elsize); + } PyGILState_Release(gil_state); set_reentrant(0); @@ -1317,9 +1328,15 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, gil_state = PyGILState_Ensure(); - TABLES_LOCK(); - res = tracemalloc_add_trace(domain, ptr, size); - TABLES_UNLOCK(); + if (tracemalloc_config.tracing) { + TABLES_LOCK(); + res = tracemalloc_add_trace(domain, ptr, size); + TABLES_UNLOCK(); + } + else { + // gh-128679: tracemalloc.stop() was called by another thread during + // PyGILState_Ensure() call. + } PyGILState_Release(gil_state); return res; From 1e3eac33065787d38c64ae8ab378fcae4e103a5b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 10 Jan 2025 15:48:16 +0100 Subject: [PATCH 02/11] Define 'res' in PyTraceMalloc_Track() --- Python/tracemalloc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 51f65c7e13857c..b4f8f8357468eb 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -1336,6 +1336,7 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, else { // gh-128679: tracemalloc.stop() was called by another thread during // PyGILState_Ensure() call. + res = 0; } PyGILState_Release(gil_state); From 59fd8de85c86df81d319c265400a760d2bfc8213 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 Jan 2025 17:01:58 +0100 Subject: [PATCH 03/11] Fix the race condition in the PyMem_RawFree() hook * Hold the table lock while calling _PyTraceMalloc_Stop(). * Add tracemalloc_raw_free(). --- Python/tracemalloc.c | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index b4f8f8357468eb..1de7f0d617a652 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -603,18 +603,39 @@ tracemalloc_realloc(void *ctx, void *ptr, size_t new_size) static void tracemalloc_free(void *ctx, void *ptr) { + if (ptr == NULL) + return; + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + alloc->free(alloc->ctx, ptr); + + TABLES_LOCK(); + REMOVE_TRACE(ptr); + TABLES_UNLOCK(); +} + + +static void +tracemalloc_raw_free(void *ctx, void *ptr) +{ if (ptr == NULL) return; + PyMemAllocatorEx *alloc = (PyMemAllocatorEx *)ctx; + /* GIL cannot be locked in PyMem_RawFree() because it would introduce a deadlock in _PyThreadState_DeleteCurrent(). */ alloc->free(alloc->ctx, ptr); TABLES_LOCK(); - REMOVE_TRACE(ptr); + if (tracemalloc_config.tracing) { + REMOVE_TRACE(ptr); + } + else { + // gh-128679: tracemalloc.stop() was called by another thread + } TABLES_UNLOCK(); } @@ -790,17 +811,15 @@ tracemalloc_clear_filename(void *value) /* reentrant flag must be set to call this function and GIL must be held */ static void -tracemalloc_clear_traces(void) +tracemalloc_clear_traces_unlocked(void) { /* The GIL protects variables against concurrent access */ assert(PyGILState_Check()); - TABLES_LOCK(); _Py_hashtable_clear(tracemalloc_traces); _Py_hashtable_clear(tracemalloc_domains); tracemalloc_traced_memory = 0; tracemalloc_peak_traced_memory = 0; - TABLES_UNLOCK(); _Py_hashtable_clear(tracemalloc_tracebacks); @@ -941,7 +960,7 @@ _PyTraceMalloc_Start(int max_nframe) alloc.malloc = tracemalloc_raw_malloc; alloc.calloc = tracemalloc_raw_calloc; alloc.realloc = tracemalloc_raw_realloc; - alloc.free = tracemalloc_free; + alloc.free = tracemalloc_raw_free; alloc.ctx = &allocators.raw; PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); @@ -974,6 +993,10 @@ _PyTraceMalloc_Stop(void) if (!tracemalloc_config.tracing) return; + // Lock to synchronize with tracemalloc_raw_free() which checks + // 'tracing' while holding the lock. + TABLES_LOCK(); + /* stop tracing Python memory allocations */ tracemalloc_config.tracing = 0; @@ -984,11 +1007,13 @@ _PyTraceMalloc_Stop(void) PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); - tracemalloc_clear_traces(); + tracemalloc_clear_traces_unlocked(); /* release memory */ raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; + + TABLES_UNLOCK(); } @@ -1436,7 +1461,9 @@ _PyTraceMalloc_ClearTraces(void) return; } set_reentrant(1); - tracemalloc_clear_traces(); + TABLES_LOCK(); + tracemalloc_clear_traces_unlocked(); + TABLES_UNLOCK(); set_reentrant(0); } From 0e6de65784dfd29945570aacd6aac056f16cd1cf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 Jan 2025 16:21:19 +0100 Subject: [PATCH 04/11] Add test --- Lib/test/test_tracemalloc.py | 5 ++ Modules/_testcapimodule.c | 92 ++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index 5755f7697de91a..ff9993522f90e4 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -1101,6 +1101,11 @@ def test_stop_untrack(self): with self.assertRaises(RuntimeError): self.untrack() + @unittest.skipIf(_testcapi is None, 'need _testcapi') + def test_tracemalloc_track_race(self): + # gh-128679: Test fix for tracemalloc.stop() race condition + _testcapi.tracemalloc_track_race() + if __name__ == "__main__": unittest.main() diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index a0a1f8af6710a3..2b864d1a9f879d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3435,6 +3435,97 @@ code_offset_to_line(PyObject* self, PyObject* const* args, Py_ssize_t nargsf) return PyLong_FromInt32(PyCode_Addr2Line(code, offset)); } + +typedef struct { + PyThread_type_lock lock; + int completed; +} tracemalloc_track_race_data; + +static void +tracemalloc_track_race_thread(void *raw_data) +{ + tracemalloc_track_race_data *data = (tracemalloc_track_race_data*)raw_data; + + PyTraceMalloc_Track(123, 10, 1); + + PyThread_acquire_lock(data->lock, 1); + data->completed += 1; + PyThread_release_lock(data->lock); +} + +// gh-128679: Test fix for tracemalloc.stop() race condition +static PyObject * +tracemalloc_track_race(PyObject *self, PyObject *args) +{ +#define NTHREAD 50 + PyObject *stop = NULL; + tracemalloc_track_race_data data = {0}; + + PyObject *tracemalloc = PyImport_ImportModule("tracemalloc"); + if (tracemalloc == NULL) { + goto error; + } + + PyObject *start = PyObject_GetAttrString(tracemalloc, "start"); + if (start == NULL) { + goto error; + } + PyObject *res = PyObject_CallFunction(start, "i", 1); + Py_DECREF(start); + if (res == NULL) { + goto error; + } + + stop = PyObject_GetAttrString(tracemalloc, "stop"); + Py_DECREF(tracemalloc); + if (stop == NULL) { + goto error; + } + + data.lock = PyThread_allocate_lock(); + if (!data.lock) { + PyErr_NoMemory(); + goto error; + } + + for (size_t i = 0; i < NTHREAD; i++) { + unsigned long thread = PyThread_start_new_thread( + tracemalloc_track_race_thread, &data); + if (thread == (unsigned long)-1) { + PyErr_SetString(PyExc_RuntimeError, "can't start new thread"); + goto error; + } + } + + res = PyObject_CallNoArgs(stop); + Py_CLEAR(stop); + if (res == NULL) { + goto error; + } + + Py_BEGIN_ALLOW_THREADS + while (1) { + PyThread_acquire_lock(data.lock, 1); + int completed = data.completed; + PyThread_release_lock(data.lock); + if (completed >= NTHREAD) { + break; + } + sleep(1); + } + Py_END_ALLOW_THREADS + + Py_RETURN_NONE; + +error: + Py_CLEAR(stop); + if (data.lock) { + PyThread_free_lock(data.lock); + } + return NULL; +#undef NTHREAD +} + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -3578,6 +3669,7 @@ static PyMethodDef TestMethods[] = { {"type_freeze", type_freeze, METH_VARARGS}, {"test_atexit", test_atexit, METH_NOARGS}, {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL}, + {"tracemalloc_track_race", tracemalloc_track_race, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; From 48e172d76fdb4f9231d1781a8791ba2ed648547d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 Jan 2025 17:21:11 +0100 Subject: [PATCH 05/11] Fix test on Windows --- Modules/_testcapimodule.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 2b864d1a9f879d..1f315d74964563 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -52,6 +52,23 @@ get_testerror(PyObject *self) { return state->error; } + +// Sleep 'ms' microseconds. +static void +pysleep_ms(int ms) +{ + assert(ms >= 1); +#ifdef MS_WINDOWS + Sleep(ms); +#else + struct timeval timeout; + timeout.tv_sec = 0; + timeout.tv_usec = ms * 1000; + (void)select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &timeout); +#endif +} + + /* Raise _testcapi.error with test_name + ": " + msg, and return NULL. */ static PyObject * @@ -3511,7 +3528,8 @@ tracemalloc_track_race(PyObject *self, PyObject *args) if (completed >= NTHREAD) { break; } - sleep(1); + + pysleep_ms(10); } Py_END_ALLOW_THREADS From 146167b0ff92adec3444eb2a072b9393529ba23a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 Jan 2025 17:23:39 +0100 Subject: [PATCH 06/11] Fix refleak on error --- Modules/_testcapimodule.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 1f315d74964563..e80fed54bde6d0 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3475,10 +3475,11 @@ static PyObject * tracemalloc_track_race(PyObject *self, PyObject *args) { #define NTHREAD 50 + PyObject *tracemalloc = NULL; PyObject *stop = NULL; tracemalloc_track_race_data data = {0}; - PyObject *tracemalloc = PyImport_ImportModule("tracemalloc"); + tracemalloc = PyImport_ImportModule("tracemalloc"); if (tracemalloc == NULL) { goto error; } @@ -3494,7 +3495,7 @@ tracemalloc_track_race(PyObject *self, PyObject *args) } stop = PyObject_GetAttrString(tracemalloc, "stop"); - Py_DECREF(tracemalloc); + Py_CLEAR(tracemalloc); if (stop == NULL) { goto error; } @@ -3536,6 +3537,7 @@ tracemalloc_track_race(PyObject *self, PyObject *args) Py_RETURN_NONE; error: + Py_CLEAR(tracemalloc); Py_CLEAR(stop); if (data.lock) { PyThread_free_lock(data.lock); From aceff9bd0b99339e87ba9964439264754624adeb Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 Jan 2025 17:33:57 +0100 Subject: [PATCH 07/11] test: add #include --- Modules/_testcapimodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index e80fed54bde6d0..0c196aab15e29d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,10 +18,13 @@ #include // FLT_MAX #include #include // offsetof() - #ifdef HAVE_SYS_WAIT_H # include // W_STOPCODE #endif +#ifdef MS_WINDOWS +# define WIN32_LEAN_AND_MEAN +# include // Sleep() +#endif #ifdef bool # error "The public headers should not include , see gh-48924" From 61edb4cfa18500d37633df66bfe61ec973786917 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 13 Jan 2025 21:17:31 +0100 Subject: [PATCH 08/11] Skip test on WASI --- Lib/test/test_tracemalloc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_tracemalloc.py b/Lib/test/test_tracemalloc.py index ff9993522f90e4..da2db28775578a 100644 --- a/Lib/test/test_tracemalloc.py +++ b/Lib/test/test_tracemalloc.py @@ -7,8 +7,9 @@ from test.support.script_helper import (assert_python_ok, assert_python_failure, interpreter_requires_environment) from test import support -from test.support import os_helper from test.support import force_not_colorized +from test.support import os_helper +from test.support import threading_helper try: import _testcapi @@ -952,7 +953,6 @@ def check_env_var_invalid(self, nframe): return self.fail(f"unexpected output: {stderr!a}") - def test_env_var_invalid(self): for nframe in INVALID_NFRAME: with self.subTest(nframe=nframe): @@ -1102,6 +1102,7 @@ def test_stop_untrack(self): self.untrack() @unittest.skipIf(_testcapi is None, 'need _testcapi') + @threading_helper.requires_working_threading() def test_tracemalloc_track_race(self): # gh-128679: Test fix for tracemalloc.stop() race condition _testcapi.tracemalloc_track_race() From 9df058a7c945c99eff1a16286f70348cef4d9849 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 16:46:24 +0100 Subject: [PATCH 09/11] Rewrite the test without sleep --- Modules/_testcapimodule.c | 88 ++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 53 deletions(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 0c196aab15e29d..7c3a7eb508e9b3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -21,10 +21,6 @@ #ifdef HAVE_SYS_WAIT_H # include // W_STOPCODE #endif -#ifdef MS_WINDOWS -# define WIN32_LEAN_AND_MEAN -# include // Sleep() -#endif #ifdef bool # error "The public headers should not include , see gh-48924" @@ -56,22 +52,6 @@ get_testerror(PyObject *self) { } -// Sleep 'ms' microseconds. -static void -pysleep_ms(int ms) -{ - assert(ms >= 1); -#ifdef MS_WINDOWS - Sleep(ms); -#else - struct timeval timeout; - timeout.tv_sec = 0; - timeout.tv_usec = ms * 1000; - (void)select(0, (fd_set *)0, (fd_set *)0, (fd_set *)0, &timeout); -#endif -} - - /* Raise _testcapi.error with test_name + ": " + msg, and return NULL. */ static PyObject * @@ -3456,21 +3436,13 @@ code_offset_to_line(PyObject* self, PyObject* const* args, Py_ssize_t nargsf) } -typedef struct { - PyThread_type_lock lock; - int completed; -} tracemalloc_track_race_data; - static void -tracemalloc_track_race_thread(void *raw_data) +tracemalloc_track_race_thread(void *data) { - tracemalloc_track_race_data *data = (tracemalloc_track_race_data*)raw_data; - PyTraceMalloc_Track(123, 10, 1); - PyThread_acquire_lock(data->lock, 1); - data->completed += 1; - PyThread_release_lock(data->lock); + PyThread_type_lock lock = (PyThread_type_lock)data; + PyThread_release_lock(lock); } // gh-128679: Test fix for tracemalloc.stop() race condition @@ -3480,18 +3452,19 @@ tracemalloc_track_race(PyObject *self, PyObject *args) #define NTHREAD 50 PyObject *tracemalloc = NULL; PyObject *stop = NULL; - tracemalloc_track_race_data data = {0}; + PyThread_type_lock locks[NTHREAD]; + memset(locks, 0, sizeof(locks)); + // Call tracemalloc.start() tracemalloc = PyImport_ImportModule("tracemalloc"); if (tracemalloc == NULL) { goto error; } - PyObject *start = PyObject_GetAttrString(tracemalloc, "start"); if (start == NULL) { goto error; } - PyObject *res = PyObject_CallFunction(start, "i", 1); + PyObject *res = PyObject_CallNoArgs(start); Py_DECREF(start); if (res == NULL) { goto error; @@ -3503,47 +3476,56 @@ tracemalloc_track_race(PyObject *self, PyObject *args) goto error; } - data.lock = PyThread_allocate_lock(); - if (!data.lock) { - PyErr_NoMemory(); - goto error; - } - + // Start threads for (size_t i = 0; i < NTHREAD; i++) { - unsigned long thread = PyThread_start_new_thread( - tracemalloc_track_race_thread, &data); + PyThread_type_lock lock = PyThread_allocate_lock(); + if (!lock) { + PyErr_NoMemory(); + goto error; + } + locks[i] = lock; + PyThread_acquire_lock(lock, 1); + + unsigned long thread; + thread = PyThread_start_new_thread(tracemalloc_track_race_thread, + (void*)lock); if (thread == (unsigned long)-1) { PyErr_SetString(PyExc_RuntimeError, "can't start new thread"); goto error; } } + // Call tracemalloc.stop() while threads are running res = PyObject_CallNoArgs(stop); Py_CLEAR(stop); if (res == NULL) { goto error; } + // Wait until threads complete with the GIL released Py_BEGIN_ALLOW_THREADS - while (1) { - PyThread_acquire_lock(data.lock, 1); - int completed = data.completed; - PyThread_release_lock(data.lock); - if (completed >= NTHREAD) { - break; - } - - pysleep_ms(10); + for (size_t i = 0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_acquire_lock(lock, 1); + PyThread_release_lock(lock); } Py_END_ALLOW_THREADS + // Free threads locks + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + PyThread_free_lock(lock); + } Py_RETURN_NONE; error: Py_CLEAR(tracemalloc); Py_CLEAR(stop); - if (data.lock) { - PyThread_free_lock(data.lock); + for (size_t i=0; i < NTHREAD; i++) { + PyThread_type_lock lock = locks[i]; + if (lock) { + PyThread_free_lock(lock); + } } return NULL; #undef NTHREAD From ecfa30b96716e3fa0ed8930d17d76a1bc7f7224b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 14 Jan 2025 16:48:17 +0100 Subject: [PATCH 10/11] revert unwanted change --- Modules/_testcapimodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 7c3a7eb508e9b3..bc492079c4b615 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -18,6 +18,7 @@ #include // FLT_MAX #include #include // offsetof() + #ifdef HAVE_SYS_WAIT_H # include // W_STOPCODE #endif @@ -51,7 +52,6 @@ get_testerror(PyObject *self) { return state->error; } - /* Raise _testcapi.error with test_name + ": " + msg, and return NULL. */ static PyObject * From 997148b227281d5e40b3f7c98c9f0c8b4cb9c98b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 15 Jan 2025 08:23:45 +0100 Subject: [PATCH 11/11] Fix refleak in the test --- Modules/_testcapimodule.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index bc492079c4b615..7d304add5999d1 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3469,6 +3469,7 @@ tracemalloc_track_race(PyObject *self, PyObject *args) if (res == NULL) { goto error; } + Py_DECREF(res); stop = PyObject_GetAttrString(tracemalloc, "stop"); Py_CLEAR(tracemalloc); @@ -3501,6 +3502,7 @@ tracemalloc_track_race(PyObject *self, PyObject *args) if (res == NULL) { goto error; } + Py_DECREF(res); // Wait until threads complete with the GIL released Py_BEGIN_ALLOW_THREADS