From cf2008871b6b34f3e96da34409aecea15c791c2e Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Thu, 26 Jan 2023 11:14:09 +0000 Subject: [PATCH 1/3] fixup ignore inconsistency --- doc/source/whatsnew/v2.0.0.rst | 1 + pandas/_libs/tslib.pyx | 60 -------------------------- pandas/tests/tools/test_to_datetime.py | 2 +- 3 files changed, 2 insertions(+), 61 deletions(-) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index 613a1e82d461f..ca830af323cbb 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -982,6 +982,7 @@ Datetimelike - Bug in :func:`to_datetime` was raising ``ValueError`` when parsing mixed-offset :class:`Timestamp` with ``errors='ignore'`` (:issue:`50585`) - Bug in :func:`to_datetime` was incorrectly handling floating-point inputs within 1 ``unit`` of the overflow boundaries (:issue:`50183`) - Bug in :func:`to_datetime` with unit of "Y" or "M" giving incorrect results, not matching pointwise :class:`Timestamp` results (:issue:`50870`) +- Bug in :func:`to_datetime` was not returning input with ``errors='ignore'`` when passed out-of-bounds np.datetime64 (:issue:`50587`) - Timedelta diff --git a/pandas/_libs/tslib.pyx b/pandas/_libs/tslib.pyx index 9d9b93f274c60..7410ea50effd1 100644 --- a/pandas/_libs/tslib.pyx +++ b/pandas/_libs/tslib.pyx @@ -34,7 +34,6 @@ from pandas._libs.tslibs.np_datetime cimport ( NPY_DATETIMEUNIT, NPY_FR_ns, check_dts_bounds, - get_datetime64_value, npy_datetimestruct, npy_datetimestruct_to_datetime, pandas_datetime_to_datetimestruct, @@ -542,16 +541,6 @@ cpdef array_to_datetime( cnp.PyArray_MultiIter_NEXT(mi) - except OutOfBoundsDatetime as ex: - ex.args = (f"{ex}, at position {i}",) - if is_coerce: - iresult[i] = NPY_NAT - cnp.PyArray_MultiIter_NEXT(mi) - continue - elif is_raise: - raise - return ignore_errors_out_of_bounds_fallback(values), tz_out - except (TypeError, OverflowError, ValueError) as ex: ex.args = (f"{ex}, at position {i}",) if is_coerce: @@ -578,55 +567,6 @@ cpdef array_to_datetime( return result, tz_out -@cython.wraparound(False) -@cython.boundscheck(False) -cdef ndarray ignore_errors_out_of_bounds_fallback(ndarray values): - """ - Fallback for array_to_datetime if an OutOfBoundsDatetime is raised - and errors == "ignore" - - Parameters - ---------- - values : ndarray[object] - - Returns - ------- - ndarray[object] - """ - cdef: - Py_ssize_t i, n = values.size - object val - cnp.broadcast mi - ndarray[object] oresult - ndarray oresult_nd - - oresult_nd = cnp.PyArray_EMPTY(values.ndim, values.shape, cnp.NPY_OBJECT, 0) - mi = cnp.PyArray_MultiIterNew2(oresult_nd, values) - oresult = oresult_nd.ravel() - - for i in range(n): - # Analogous to `val = values[i]` - val = (cnp.PyArray_MultiIter_DATA(mi, 1))[0] - - # set as nan except if its a NaT - if checknull_with_nat_and_na(val): - if isinstance(val, float): - oresult[i] = np.nan - else: - oresult[i] = NaT - elif is_datetime64_object(val): - if get_datetime64_value(val) == NPY_NAT: - oresult[i] = NaT - else: - oresult[i] = val.item() - else: - oresult[i] = val - - cnp.PyArray_MultiIter_NEXT(mi) - - return oresult - - @cython.wraparound(False) @cython.boundscheck(False) cdef _array_to_datetime_object( diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index a1217b268613a..dbfbe87b9448a 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -1084,7 +1084,7 @@ def test_to_datetime_array_of_dt64s(self, cache, unit): # numpy is either a python datetime.datetime or datetime.date tm.assert_index_equal( to_datetime(dts_with_oob, errors="ignore", cache=cache), - Index([dt.item() for dt in dts_with_oob]), + Index(dts_with_oob), ) def test_to_datetime_tz(self, cache): From ea43a199a2e2db2f290e33c4901b760c32507462 Mon Sep 17 00:00:00 2001 From: MarcoGorelli <> Date: Tue, 31 Jan 2023 20:31:30 +0000 Subject: [PATCH 2/3] add unit test, clarify whatsnew --- doc/source/whatsnew/v2.0.0.rst | 2 +- pandas/tests/tools/test_to_datetime.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index ca830af323cbb..2a1126356c164 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -982,7 +982,7 @@ Datetimelike - Bug in :func:`to_datetime` was raising ``ValueError`` when parsing mixed-offset :class:`Timestamp` with ``errors='ignore'`` (:issue:`50585`) - Bug in :func:`to_datetime` was incorrectly handling floating-point inputs within 1 ``unit`` of the overflow boundaries (:issue:`50183`) - Bug in :func:`to_datetime` with unit of "Y" or "M" giving incorrect results, not matching pointwise :class:`Timestamp` results (:issue:`50870`) -- Bug in :func:`to_datetime` was not returning input with ``errors='ignore'`` when passed out-of-bounds np.datetime64 (:issue:`50587`) +- Bug in :func:`to_datetime` was not returning input with ``errors='ignore'`` when input was out-of-bounds (:issue:`50587`) - Timedelta diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index dbfbe87b9448a..3f86cfc2830c1 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -1087,6 +1087,12 @@ def test_to_datetime_array_of_dt64s(self, cache, unit): Index(dts_with_oob), ) + def test_out_of_bounds_errors_ignore(self): + # https://github.com/pandas-dev/pandas/issues/50587 + result = to_datetime([np.datetime64("9999-01-01")], errors="ignore")[0] + expected = np.datetime64("9999-01-01") + assert result == expected + def test_to_datetime_tz(self, cache): # xref 8260 From 3185872552a6a189a0b30b49a46c0333ac634565 Mon Sep 17 00:00:00 2001 From: Marco Edward Gorelli <33491632+MarcoGorelli@users.noreply.github.com> Date: Thu, 2 Feb 2023 09:24:50 +0000 Subject: [PATCH 3/3] simplify Co-authored-by: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> --- pandas/tests/tools/test_to_datetime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/tools/test_to_datetime.py b/pandas/tests/tools/test_to_datetime.py index 3f86cfc2830c1..7a93d2fe8b5ce 100644 --- a/pandas/tests/tools/test_to_datetime.py +++ b/pandas/tests/tools/test_to_datetime.py @@ -1089,7 +1089,7 @@ def test_to_datetime_array_of_dt64s(self, cache, unit): def test_out_of_bounds_errors_ignore(self): # https://github.com/pandas-dev/pandas/issues/50587 - result = to_datetime([np.datetime64("9999-01-01")], errors="ignore")[0] + result = to_datetime(np.datetime64("9999-01-01"), errors="ignore") expected = np.datetime64("9999-01-01") assert result == expected