diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 13f86b01bbfe8f..74685a3e945459 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -70,8 +70,13 @@ extern PyObject* _Py_device_encoding(int); # define _PY_WRITE_MAX INT_MAX #else /* write() should truncate the input to PY_SSIZE_T_MAX bytes, - but it's safer to do it ourself to have a portable behaviour */ -# define _PY_READ_MAX PY_SSIZE_T_MAX + but it's safer to do it ourself to have a portable behaviour + + read() fills a PyBytes object, which has a capped size defined in + bytesobject.c. Prefer reading less data (meets the API spec) to causing + an overflow error. + */ +# define _PY_READ_MAX PY_SSIZE_T_MAX - 4096 # define _PY_WRITE_MAX PY_SSIZE_T_MAX #endif diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f93937fb587386..6c0bb6a2cfc60c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -234,10 +234,6 @@ def test_read(self): self.assertEqual(s, b"spam") @support.cpython_only - # Skip the test on 32-bit platforms: the number of bytes must fit in a - # Py_ssize_t type - @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, - "needs INT_MAX < PY_SSIZE_T_MAX") @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) def test_large_read(self, size): self.addCleanup(os_helper.unlink, os_helper.TESTFN) @@ -245,6 +241,9 @@ def test_large_read(self, size): # Issue #21932: Make sure that os.read() does not raise an # OverflowError for size larger than INT_MAX + # + # For 32 bit systems, it is expected the read doesn't read all the bytes + # but rather caps to a number it can reasonably read. with open(os_helper.TESTFN, "rb") as fp: data = os.read(fp.fileno(), size) diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-07-11-13-57-50.gh-issue-120754.C1HedA.rst b/Misc/NEWS.d/next/Core and Builtins/2024-07-11-13-57-50.gh-issue-120754.C1HedA.rst new file mode 100644 index 00000000000000..47b50360225075 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-07-11-13-57-50.gh-issue-120754.C1HedA.rst @@ -0,0 +1,3 @@ +``os.read()`` caps read size smaller than the max bytes object size to avoid +getting an OverflowError that 'byte string is too large'. This makes it so the +read call is attempted (although still may fail for other reasons).