From 979dd2be21e4b6c2727e259f0e3739e95f854121 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Tue, 7 Mar 2023 11:25:54 +0100 Subject: [PATCH 1/5] gh-102494: fix MemoryError when using selectors on Solaris --- Modules/selectmodule.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index d234d504cb5167..381674b9d53f94 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1133,7 +1133,7 @@ newDevPollObject(PyObject *module) struct rlimit limit; /* - ** If we try to process more that getrlimit() + ** If we try to process more than getrlimit() ** fds, the kernel will give an error, so ** we set the limit here. It is a dynamic ** value, because we can change rlimit() anytime. @@ -1144,6 +1144,15 @@ newDevPollObject(PyObject *module) return NULL; } + /* + ** If the limit is too high (or RLIM_INFINITY), we might allocate huge + ** amounts of memory (or even fail to allocate). Because of that, we limit + ** the number of allocated structs to 2^18 (which is ~4MB of memory). + */ + if (limit.rlim_cur > (rlim_t)262144) { + limit.rlim_cur = (rlim_t)262144; + } + fd_devpoll = _Py_open("/dev/poll", O_RDWR); if (fd_devpoll == -1) return NULL; From 7db3a64a914ca55ecfc72bfe417dd34fef97a351 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Tue, 7 Mar 2023 10:41:41 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst diff --git a/Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst b/Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst new file mode 100644 index 00000000000000..7439dc8fbea2da --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst @@ -0,0 +1 @@ +The devpoll object now has an upper limit on number of registered file descriptors of 2^18 to prevent memory exhaustion at the time of instantiation. From f949dea21f419f431f196796925a1231a1c0e697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Wed, 30 Jul 2025 14:13:25 +0200 Subject: [PATCH 3/5] split the in and out buffer into two with different sizes --- Modules/selectmodule.c | 49 +++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 381674b9d53f94..e226ecbb4ac637 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -790,12 +790,16 @@ poll_dealloc(PyObject *op) #ifdef HAVE_SYS_DEVPOLL_H static PyMethodDef devpoll_methods[]; +#define DEVPOLL_IN_BUFFER_SIZE 128 +#define DEVPOLL_MAX_OUT_BUFFER_SIZE 1024 + typedef struct { PyObject_HEAD int fd_devpoll; - int max_n_fds; int n_fds; + int out_size; struct pollfd *fds; + struct pollfd *out_fds; } devpollObject; #define devpollObject_CAST(op) ((devpollObject *)(op)) @@ -848,7 +852,7 @@ internal_devpoll_register(devpollObject *self, int fd, self->fds[self->n_fds].fd = fd; self->fds[self->n_fds].events = POLLREMOVE; - if (++self->n_fds == self->max_n_fds) { + if (++self->n_fds == DEVPOLL_IN_BUFFER_SIZE) { if (devpoll_flush(self)) return NULL; } @@ -857,7 +861,7 @@ internal_devpoll_register(devpollObject *self, int fd, self->fds[self->n_fds].fd = fd; self->fds[self->n_fds].events = (signed short)events; - if (++self->n_fds == self->max_n_fds) { + if (++self->n_fds == DEVPOLL_IN_BUFFER_SIZE) { if (devpoll_flush(self)) return NULL; } @@ -929,7 +933,7 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) self->fds[self->n_fds].fd = fd; self->fds[self->n_fds].events = POLLREMOVE; - if (++self->n_fds == self->max_n_fds) { + if (++self->n_fds == DEVPOLL_IN_BUFFER_SIZE) { if (devpoll_flush(self)) return NULL; } @@ -989,8 +993,8 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) if (devpoll_flush(self)) return NULL; - dvp.dp_fds = self->fds; - dvp.dp_nfds = self->max_n_fds; + dvp.dp_fds = self->out_fds; + dvp.dp_nfds = self->out_size; dvp.dp_timeout = (int)ms; if (timeout >= 0) { @@ -1034,8 +1038,8 @@ select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) return NULL; for (i = 0; i < poll_result; i++) { - num1 = PyLong_FromLong(self->fds[i].fd); - num2 = PyLong_FromLong(self->fds[i].revents); + num1 = PyLong_FromLong(self->out_fds[i].fd); + num2 = PyLong_FromLong(self->out_fds[i].revents); if ((num1 == NULL) || (num2 == NULL)) { Py_XDECREF(num1); Py_XDECREF(num2); @@ -1128,8 +1132,8 @@ static devpollObject * newDevPollObject(PyObject *module) { devpollObject *self; - int fd_devpoll, limit_result; - struct pollfd *fds; + int fd_devpoll, limit_result, out_size; + struct pollfd *fds, *out_fds; struct rlimit limit; /* @@ -1145,35 +1149,45 @@ newDevPollObject(PyObject *module) } /* - ** If the limit is too high (or RLIM_INFINITY), we might allocate huge - ** amounts of memory (or even fail to allocate). Because of that, we limit - ** the number of allocated structs to 2^18 (which is ~4MB of memory). + ** If the limit is too high (or RLIM_INFINITY), we might + ** allocate huge amounts of memory or even fail to allocate. */ - if (limit.rlim_cur > (rlim_t)262144) { - limit.rlim_cur = (rlim_t)262144; + out_size = limit.rlim_cur; + if (out_size > DEVPOLL_MAX_OUT_BUFFER_SIZE) { + out_size = DEVPOLL_MAX_OUT_BUFFER_SIZE; } fd_devpoll = _Py_open("/dev/poll", O_RDWR); if (fd_devpoll == -1) return NULL; - fds = PyMem_NEW(struct pollfd, limit.rlim_cur); + fds = PyMem_NEW(struct pollfd, DEVPOLL_IN_BUFFER_SIZE); if (fds == NULL) { close(fd_devpoll); PyErr_NoMemory(); return NULL; } + out_fds = PyMem_NEW(struct pollfd, out_size); + if (fds == NULL) { + close(fd_devpoll); + PyMem_Free(fds); + PyErr_NoMemory(); + return NULL; + } + self = PyObject_New(devpollObject, get_select_state(module)->devpoll_Type); if (self == NULL) { close(fd_devpoll); PyMem_Free(fds); + PyMem_Free(out_fds); return NULL; } self->fd_devpoll = fd_devpoll; - self->max_n_fds = limit.rlim_cur; + self->max_n_fds = out_size; self->n_fds = 0; self->fds = fds; + self->out_fds = out_fds; return self; } @@ -1185,6 +1199,7 @@ devpoll_dealloc(PyObject *op) PyTypeObject *type = Py_TYPE(self); (void)devpoll_internal_close(self); PyMem_Free(self->fds); + PyMem_Free(self->out_fds); PyObject_Free(self); Py_DECREF(type); } From 86100531ef348810ccdb7714ef1a47e82d167fbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Wed, 30 Jul 2025 14:16:40 +0200 Subject: [PATCH 4/5] remove the old blurb --- .../next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst | 1 - 1 file changed, 1 deletion(-) delete mode 100644 Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst diff --git a/Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst b/Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst deleted file mode 100644 index 7439dc8fbea2da..00000000000000 --- a/Misc/NEWS.d/next/Library/2023-03-07-10-41-37.gh-issue-102494.-fBwpb.rst +++ /dev/null @@ -1 +0,0 @@ -The devpoll object now has an upper limit on number of registered file descriptors of 2^18 to prevent memory exhaustion at the time of instantiation. From adf2e6f5868c1472fc2a98a990877532368c9e40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Wed, 30 Jul 2025 14:47:07 +0200 Subject: [PATCH 5/5] minor fix --- Modules/selectmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index e226ecbb4ac637..327a079b5e97a3 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -1184,9 +1184,9 @@ newDevPollObject(PyObject *module) return NULL; } self->fd_devpoll = fd_devpoll; - self->max_n_fds = out_size; self->n_fds = 0; self->fds = fds; + self->out_size = out_size; self->out_fds = out_fds; return self;