From ce397ae89f26ed7cdad8ff2b63e4ab84b8b2c258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:03:56 +0100 Subject: [PATCH 1/4] allow to mark codecs handlers tests as safe --- Lib/test/test_capi/test_codecs.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index a557e35e68915d..5ab8c07adce892 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -835,7 +835,8 @@ def test_codec_strict_errors_handler(self): def test_codec_ignore_errors_handler(self): handler = _testcapi.codec_ignore_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors, + safe=True) def test_codec_replace_errors_handler(self): handler = _testcapi.codec_replace_errors @@ -851,20 +852,29 @@ def test_codec_backslashreplace_errors_handler(self): def test_codec_namereplace_errors_handler(self): handler = _testlimitedcapi.codec_namereplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors, + safe=True) - def do_test_codec_errors_handler(self, handler, exceptions): + def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): at_least_one = False for exc in exceptions: # See https://github.com/python/cpython/issues/123378 and related # discussion and issues for details. - if self._exception_may_crash(exc): + if not safe and self._exception_may_crash(exc): continue at_least_one = True with self.subTest(handler=handler, exc=exc): # test that the handler does not crash - self.assertIsInstance(handler(exc), tuple) + res = handler(exc) + self.assertIsInstance(res, tuple) + self.assertEqual(len(res), 2) + self.assertIsInstance(res[0], str) + # We only check the type of res[1] but not its range since + # it depends on *_GetStart() and *_GetEnd() functions which + # automatically clip the 'start' and 'end' values (possibly + # negative or inconsistent). + self.assertIsInstance(res[1], int) if exceptions: self.assertTrue(at_least_one, "all exceptions are crashing") From 221618125288d968de163bcc64247cbfbc620738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 6 Dec 2024 17:19:49 +0100 Subject: [PATCH 2/4] upgrade tests --- Lib/test/test_capi/test_codecs.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 5ab8c07adce892..b69fd2fe87594c 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -869,12 +869,11 @@ def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): res = handler(exc) self.assertIsInstance(res, tuple) self.assertEqual(len(res), 2) - self.assertIsInstance(res[0], str) - # We only check the type of res[1] but not its range since - # it depends on *_GetStart() and *_GetEnd() functions which - # automatically clip the 'start' and 'end' values (possibly - # negative or inconsistent). - self.assertIsInstance(res[1], int) + replacement, continue_from = res + self.assertIsInstance(replacement, str) + self.assertIsInstance(continue_from, int) + self.assertGreaterEqual(continue_from, 0) + self.assertLessEqual(continue_from, len(exc.object)) if exceptions: self.assertTrue(at_least_one, "all exceptions are crashing") From d6bc869a1f24a2dd403ca608baa97c0b46c0bb67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:33:30 +0100 Subject: [PATCH 3/4] remove redundant safeguards --- Lib/test/test_capi/test_codecs.py | 53 ++++--------------------------- 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index f4e1e153a90834..1cc4afc5167094 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -835,38 +835,27 @@ def test_codec_strict_errors_handler(self): def test_codec_ignore_errors_handler(self): handler = _testcapi.codec_ignore_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) def test_codec_replace_errors_handler(self): handler = _testcapi.codec_replace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) def test_codec_xmlcharrefreplace_errors_handler(self): handler = _testcapi.codec_xmlcharrefreplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) def test_codec_backslashreplace_errors_handler(self): handler = _testcapi.codec_backslashreplace_errors - self.do_test_codec_errors_handler(handler, self.all_unicode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.all_unicode_errors) def test_codec_namereplace_errors_handler(self): handler = _testlimitedcapi.codec_namereplace_errors - self.do_test_codec_errors_handler(handler, self.unicode_encode_errors, - safe=True) + self.do_test_codec_errors_handler(handler, self.unicode_encode_errors) - def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): - at_least_one = False + def do_test_codec_errors_handler(self, handler, exceptions): + self.assertNotEqual(len(exceptions), 0) for exc in exceptions: - # See https://github.com/python/cpython/issues/123378 and related - # discussion and issues for details. - if not safe and self._exception_may_crash(exc): - continue - - at_least_one = True with self.subTest(handler=handler, exc=exc): # test that the handler does not crash res = handler(exc) @@ -878,9 +867,6 @@ def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): self.assertGreaterEqual(continue_from, 0) self.assertLessEqual(continue_from, len(exc.object)) - if exceptions: - self.assertTrue(at_least_one, "all exceptions are crashing") - for bad_exc in ( self.bad_unicode_errors + tuple(e for e in self.all_unicode_errors if e not in exceptions) @@ -888,30 +874,5 @@ def do_test_codec_errors_handler(self, handler, exceptions, *, safe=False): with self.subTest('bad type', handler=handler, exc=bad_exc): self.assertRaises(TypeError, handler, bad_exc) - @classmethod - def _exception_may_crash(cls, exc): - """Indicate whether a Unicode exception might currently crash - the interpreter when used by a built-in codecs error handler. - - Until gh-123378 is fixed, we skip the tests for these exceptions. - - This should only be used by "do_test_codec_errors_handler". - """ - message, start, end = exc.object, exc.start, exc.end - match exc: - case UnicodeEncodeError(): - return end < start or (end - start) >= len(message) - case UnicodeDecodeError(): - # The case "end - start >= len(message)" does not crash. - return end < start - case UnicodeTranslateError(): - # Test "end <= start" because PyCodec_ReplaceErrors checks - # the Unicode kind of a 0-length string which by convention - # is PyUnicode_1BYTE_KIND and not PyUnicode_2BYTE_KIND as - # the handler currently expects. - return end <= start or (end - start) >= len(message) - return False - - if __name__ == "__main__": unittest.main() From 2e7f3231382c1ef84a9e20a27baf7b8a4d135dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 23 Jan 2025 14:34:01 +0100 Subject: [PATCH 4/4] add double blank line --- Lib/test/test_capi/test_codecs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_capi/test_codecs.py b/Lib/test/test_capi/test_codecs.py index 1cc4afc5167094..a0355c7a388c57 100644 --- a/Lib/test/test_capi/test_codecs.py +++ b/Lib/test/test_capi/test_codecs.py @@ -874,5 +874,6 @@ def do_test_codec_errors_handler(self, handler, exceptions): with self.subTest('bad type', handler=handler, exc=bad_exc): self.assertRaises(TypeError, handler, bad_exc) + if __name__ == "__main__": unittest.main()