From 13dfb33fd6e7d40023f65634d072576462fe66fd Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 13 Feb 2023 14:26:27 -0800 Subject: [PATCH 1/4] REF: use datetime C API instead of getattrs --- .../_libs/src/ujson/python/date_conversions.c | 8 +++-- .../_libs/tslibs/src/datetime/np_datetime.c | 35 ++++++++----------- .../_libs/tslibs/src/datetime/np_datetime.h | 4 ++- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/pandas/_libs/src/ujson/python/date_conversions.c b/pandas/_libs/src/ujson/python/date_conversions.c index 86cb68f869cb0..f337c393a2cc0 100644 --- a/pandas/_libs/src/ujson/python/date_conversions.c +++ b/pandas/_libs/src/ujson/python/date_conversions.c @@ -79,7 +79,9 @@ char *PyDateTimeToIso(PyObject *obj, NPY_DATETIMEUNIT base, npy_datetimestruct dts; int ret; - ret = convert_pydatetime_to_datetimestruct(obj, &dts); + PyDateTime_IMPORT; + + ret = convert_pydatetime_to_datetimestruct((PyDateTime_Date*)obj, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, @@ -121,7 +123,9 @@ npy_datetime PyDateTimeToEpoch(PyObject *dt, NPY_DATETIMEUNIT base) { npy_datetimestruct dts; int ret; - ret = convert_pydatetime_to_datetimestruct(dt, &dts); + PyDateTime_IMPORT; + + ret = convert_pydatetime_to_datetimestruct((PyDateTime_Date*)dt, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.c b/pandas/_libs/tslibs/src/datetime/np_datetime.c index 2bac6c720c3b6..8a4a5f50594dc 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.c +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.c @@ -27,6 +27,7 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt #include #include "np_datetime.h" +#include "datetime.h" const npy_datetimestruct _AS_MIN_DTS = { 1969, 12, 31, 23, 59, 50, 776627, 963145, 224193}; @@ -370,39 +371,31 @@ PyObject *extract_utc_offset(PyObject *obj) { * Returns -1 on error, 0 on success, and 1 (with no error set) * if obj doesn't have the needed date or datetime attributes. */ -int convert_pydatetime_to_datetimestruct(PyObject *dtobj, +int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, npy_datetimestruct *out) { // Assumes that obj is a valid datetime object PyObject *tmp; - PyObject *obj = (PyObject*)dtobj; /* Initialize the output to all zeros */ memset(out, 0, sizeof(npy_datetimestruct)); out->month = 1; out->day = 1; - out->year = PyLong_AsLong(PyObject_GetAttrString(obj, "year")); - out->month = PyLong_AsLong(PyObject_GetAttrString(obj, "month")); - out->day = PyLong_AsLong(PyObject_GetAttrString(obj, "day")); + PyDateTime_IMPORT; - // TODO(anyone): If we can get PyDateTime_IMPORT to work, we could use - // PyDateTime_Check here, and less verbose attribute lookups. + out->year = PyDateTime_GET_YEAR(dtobj); + out->month = PyDateTime_GET_MONTH(dtobj); + out->day = PyDateTime_GET_DAY(dtobj); + out->hour = PyDateTime_DATE_GET_HOUR(dtobj); - /* Check for time attributes (if not there, return success as a date) */ - if (!PyObject_HasAttrString(obj, "hour") || - !PyObject_HasAttrString(obj, "minute") || - !PyObject_HasAttrString(obj, "second") || - !PyObject_HasAttrString(obj, "microsecond")) { - return 0; - } - - out->hour = PyLong_AsLong(PyObject_GetAttrString(obj, "hour")); - out->min = PyLong_AsLong(PyObject_GetAttrString(obj, "minute")); - out->sec = PyLong_AsLong(PyObject_GetAttrString(obj, "second")); - out->us = PyLong_AsLong(PyObject_GetAttrString(obj, "microsecond")); + if (PyDateTime_Check(dtobj)) { + PyDateTime_DateTime* obj = (PyDateTime_DateTime*)dtobj; + out->min = PyDateTime_DATE_GET_MINUTE(obj); + out->sec = PyDateTime_DATE_GET_SECOND(obj); + out->us = PyDateTime_DATE_GET_MICROSECOND(obj); - if (PyObject_HasAttrString(obj, "tzinfo")) { - PyObject *offset = extract_utc_offset(obj); + // TODO(py3.10): in py3.10 we can use PyDateTime_DATE_GET_TZINFO + PyObject *offset = extract_utc_offset((PyObject*)obj); /* Apply the time zone offset if datetime obj is tz-aware */ if (offset != NULL) { if (offset == Py_None) { diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.h b/pandas/_libs/tslibs/src/datetime/np_datetime.h index 6ab915e517cfb..64b677a10f704 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.h +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.h @@ -23,6 +23,8 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt #include +#include "datetime.h" + typedef struct { npy_int64 days; npy_int32 hrs, min, sec, ms, us, ns, seconds, microseconds, nanoseconds; @@ -50,7 +52,7 @@ extern const npy_datetimestruct _M_MAX_DTS; PyObject *extract_utc_offset(PyObject *obj); -int convert_pydatetime_to_datetimestruct(PyObject *dtobj, +int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, npy_datetimestruct *out); npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base, From 494c918325d00994858bd5eafd30c3cdc09b0ca0 Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 13 Feb 2023 17:43:11 -0800 Subject: [PATCH 2/4] type checking --- pandas/_libs/src/ujson/python/date_conversions.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pandas/_libs/src/ujson/python/date_conversions.c b/pandas/_libs/src/ujson/python/date_conversions.c index f337c393a2cc0..498b3f4e0914a 100644 --- a/pandas/_libs/src/ujson/python/date_conversions.c +++ b/pandas/_libs/src/ujson/python/date_conversions.c @@ -80,8 +80,13 @@ char *PyDateTimeToIso(PyObject *obj, NPY_DATETIMEUNIT base, int ret; PyDateTime_IMPORT; + if (!PyDate_Check(obj)) { + PyErr_SetString(PyExc_TypeError, "Expected date object"); + return NULL; + } + PyDateTime_Date *dtobj = (PyDateTime_Date*)obj; - ret = convert_pydatetime_to_datetimestruct((PyDateTime_Date*)obj, &dts); + ret = convert_pydatetime_to_datetimestruct(dtobj, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, @@ -125,7 +130,13 @@ npy_datetime PyDateTimeToEpoch(PyObject *dt, NPY_DATETIMEUNIT base) { PyDateTime_IMPORT; - ret = convert_pydatetime_to_datetimestruct((PyDateTime_Date*)dt, &dts); + if (!PyDate_Check(dt)) { + PyErr_SetString(PyExc_TypeError, "Expected date object"); + return NULL; + } + PyDateTime_Date *dtobj = (PyDateTime_Date*)dt; + + ret = convert_pydatetime_to_datetimestruct(dtobj, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, From 21548220c2d2704282c2b492e72d778ae21b26ce Mon Sep 17 00:00:00 2001 From: Brock Date: Mon, 13 Feb 2023 19:21:50 -0800 Subject: [PATCH 3/4] troubleshoot CI --- pandas/_libs/src/ujson/python/date_conversions.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/_libs/src/ujson/python/date_conversions.c b/pandas/_libs/src/ujson/python/date_conversions.c index 498b3f4e0914a..db6dc0fc90421 100644 --- a/pandas/_libs/src/ujson/python/date_conversions.c +++ b/pandas/_libs/src/ujson/python/date_conversions.c @@ -132,7 +132,7 @@ npy_datetime PyDateTimeToEpoch(PyObject *dt, NPY_DATETIMEUNIT base) { if (!PyDate_Check(dt)) { PyErr_SetString(PyExc_TypeError, "Expected date object"); - return NULL; + // return NULL; } PyDateTime_Date *dtobj = (PyDateTime_Date*)dt; From d3bfd4966b5bced54fd795dd80ddae4f418aa11d Mon Sep 17 00:00:00 2001 From: Brock Date: Tue, 14 Feb 2023 15:53:05 -0800 Subject: [PATCH 4/4] Simplify --- .../_libs/src/ujson/python/date_conversions.c | 19 ++---------------- .../_libs/tslibs/src/datetime/np_datetime.c | 20 ++++++++++++------- .../_libs/tslibs/src/datetime/np_datetime.h | 4 +--- 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/pandas/_libs/src/ujson/python/date_conversions.c b/pandas/_libs/src/ujson/python/date_conversions.c index db6dc0fc90421..86cb68f869cb0 100644 --- a/pandas/_libs/src/ujson/python/date_conversions.c +++ b/pandas/_libs/src/ujson/python/date_conversions.c @@ -79,14 +79,7 @@ char *PyDateTimeToIso(PyObject *obj, NPY_DATETIMEUNIT base, npy_datetimestruct dts; int ret; - PyDateTime_IMPORT; - if (!PyDate_Check(obj)) { - PyErr_SetString(PyExc_TypeError, "Expected date object"); - return NULL; - } - PyDateTime_Date *dtobj = (PyDateTime_Date*)obj; - - ret = convert_pydatetime_to_datetimestruct(dtobj, &dts); + ret = convert_pydatetime_to_datetimestruct(obj, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, @@ -128,15 +121,7 @@ npy_datetime PyDateTimeToEpoch(PyObject *dt, NPY_DATETIMEUNIT base) { npy_datetimestruct dts; int ret; - PyDateTime_IMPORT; - - if (!PyDate_Check(dt)) { - PyErr_SetString(PyExc_TypeError, "Expected date object"); - // return NULL; - } - PyDateTime_Date *dtobj = (PyDateTime_Date*)dt; - - ret = convert_pydatetime_to_datetimestruct(dtobj, &dts); + ret = convert_pydatetime_to_datetimestruct(dt, &dts); if (ret != 0) { if (!PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.c b/pandas/_libs/tslibs/src/datetime/np_datetime.c index 8a4a5f50594dc..3a84adf083292 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.c +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.c @@ -371,7 +371,7 @@ PyObject *extract_utc_offset(PyObject *obj) { * Returns -1 on error, 0 on success, and 1 (with no error set) * if obj doesn't have the needed date or datetime attributes. */ -int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, +int convert_pydatetime_to_datetimestruct(PyObject *dtobj, npy_datetimestruct *out) { // Assumes that obj is a valid datetime object PyObject *tmp; @@ -383,13 +383,19 @@ int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, PyDateTime_IMPORT; - out->year = PyDateTime_GET_YEAR(dtobj); - out->month = PyDateTime_GET_MONTH(dtobj); - out->day = PyDateTime_GET_DAY(dtobj); - out->hour = PyDateTime_DATE_GET_HOUR(dtobj); + if (!PyDate_Check(dtobj)) { + PyErr_SetString(PyExc_TypeError, "Expected date object"); + return NULL; + } + PyDateTime_Date *dateobj = (PyDateTime_Date*)dtobj; + + out->year = PyDateTime_GET_YEAR(dateobj); + out->month = PyDateTime_GET_MONTH(dateobj); + out->day = PyDateTime_GET_DAY(dateobj); + out->hour = PyDateTime_DATE_GET_HOUR(dateobj); - if (PyDateTime_Check(dtobj)) { - PyDateTime_DateTime* obj = (PyDateTime_DateTime*)dtobj; + if (PyDateTime_Check(dateobj)) { + PyDateTime_DateTime* obj = (PyDateTime_DateTime*)dateobj; out->min = PyDateTime_DATE_GET_MINUTE(obj); out->sec = PyDateTime_DATE_GET_SECOND(obj); out->us = PyDateTime_DATE_GET_MICROSECOND(obj); diff --git a/pandas/_libs/tslibs/src/datetime/np_datetime.h b/pandas/_libs/tslibs/src/datetime/np_datetime.h index 64b677a10f704..6ab915e517cfb 100644 --- a/pandas/_libs/tslibs/src/datetime/np_datetime.h +++ b/pandas/_libs/tslibs/src/datetime/np_datetime.h @@ -23,8 +23,6 @@ This file is derived from NumPy 1.7. See NUMPY_LICENSE.txt #include -#include "datetime.h" - typedef struct { npy_int64 days; npy_int32 hrs, min, sec, ms, us, ns, seconds, microseconds, nanoseconds; @@ -52,7 +50,7 @@ extern const npy_datetimestruct _M_MAX_DTS; PyObject *extract_utc_offset(PyObject *obj); -int convert_pydatetime_to_datetimestruct(PyDateTime_Date *dtobj, +int convert_pydatetime_to_datetimestruct(PyObject *dtobj, npy_datetimestruct *out); npy_datetime npy_datetimestruct_to_datetime(NPY_DATETIMEUNIT base,