From 32448d68d7ff55bd392a9a60f5a048181ca80719 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Sat, 4 Jul 2020 16:36:55 -0500 Subject: [PATCH] try to port pickle to multi-stage init --- ...2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst | 1 + Modules/_pickle.c | 259 ++++++++---------- 2 files changed, 122 insertions(+), 138 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst new file mode 100644 index 00000000000000..a44047351a307f --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-07-04-09-04-41.bpo-1635741.ZsP31Y.rst @@ -0,0 +1 @@ +Port :mod:`_pickle` to multiphase initialization diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 25e888db19c235..58c8c7e0fd1b55 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -14,6 +14,9 @@ PyDoc_STRVAR(pickle_module_doc, "Optimized C implementation for the Python pickle module."); +//TODO should we remove Pickler_Type from the clinic input here? +//similar entry was not removed in pull 20960 for _bz2 + /*[clinic input] module _pickle class _pickle.Pickler "PicklerObject *" "&Pickler_Type" @@ -173,6 +176,10 @@ typedef struct { /* functools.partial, used for implementing __newobj_ex__ with protocols 2 and 3 */ PyObject *partial; + + /* needed for multi-state initialization*/ + PyTypeObject *pickler_type; + PyTypeObject *unpickler_type; } PickleState; /* Forward declaration of the _pickle module definition. */ @@ -211,6 +218,8 @@ _Pickle_ClearState(PickleState *st) Py_CLEAR(st->codecs_encode); Py_CLEAR(st->getattr); Py_CLEAR(st->partial); + Py_CLEAR(st->pickler_type); + Py_CLEAR(st->unpickler_type); } /* Initialize the given pickle module state. */ @@ -708,8 +717,6 @@ typedef struct { /* Forward declarations */ static int save(PicklerObject *, PyObject *, int); static int save_reduce(PicklerObject *, PyObject *, PyObject *); -static PyTypeObject Pickler_Type; -static PyTypeObject Unpickler_Type; #include "clinic/_pickle.c.h" @@ -1102,11 +1109,12 @@ _Pickler_Write(PicklerObject *self, const char *s, Py_ssize_t data_len) } static PicklerObject * -_Pickler_New(void) +_Pickler_New(PyObject *module) { PicklerObject *self; - self = PyObject_GC_New(PicklerObject, &Pickler_Type); + PickleState *st = _Pickle_GetState(module); + self = PyObject_GC_New(PicklerObject, st->pickler_type); if (self == NULL) return NULL; @@ -1586,11 +1594,12 @@ _Unpickler_MemoCleanup(UnpicklerObject *self) } static UnpicklerObject * -_Unpickler_New(void) +_Unpickler_New(PyObject *module) { UnpicklerObject *self; + PickleState *st = _Pickle_GetState(module); - self = PyObject_GC_New(UnpicklerObject, &Unpickler_Type); + self = PyObject_GC_New(UnpicklerObject, st->unpickler_type); if (self == NULL) return NULL; @@ -5071,47 +5080,28 @@ static PyGetSetDef Pickler_getsets[] = { {NULL} }; -static PyTypeObject Pickler_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Pickler" , /*tp_name*/ - sizeof(PicklerObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Pickler_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - _pickle_Pickler___init____doc__, /*tp_doc*/ - (traverseproc)Pickler_traverse, /*tp_traverse*/ - (inquiry)Pickler_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - Pickler_methods, /*tp_methods*/ - Pickler_members, /*tp_members*/ - Pickler_getsets, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - _pickle_Pickler___init__, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_GC_Del, /*tp_free*/ - 0, /*tp_is_gc*/ +static PyType_Slot pickle_pickler_type_slots[] = { + {Py_tp_dealloc, Pickler_dealloc}, + {Py_tp_methods, Pickler_methods}, + {Py_tp_members, Pickler_members}, + {Py_tp_getset, Pickler_getsets}, + {Py_tp_clear, Pickler_clear}, + {Py_tp_doc, (char*)_pickle_Pickler___init____doc__}, + {Py_tp_traverse, Pickler_traverse}, + {0, 0} +}; + +//TODO - how to handle this?? +// Calling PyType_GetModuleState() on a subclass is not safe. +// bz2_compressor_type_spec does not have Py_TPFLAGS_BASETYPE flag +// which prevents to create a subclass. +// So calling PyType_GetModuleState() in this file is always safe. + +static PyType_Spec pickle_pickler_type_spec = { + .name = "_pickle.Pickler", + .basicsize = sizeof(PicklerObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = pickle_pickler_type_slots, }; /* Temporary helper for calling self.find_class(). @@ -7623,47 +7613,21 @@ static PyGetSetDef Unpickler_getsets[] = { {NULL} }; -static PyTypeObject Unpickler_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "_pickle.Unpickler", /*tp_name*/ - sizeof(UnpicklerObject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - (destructor)Unpickler_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - _pickle_Unpickler___init____doc__, /*tp_doc*/ - (traverseproc)Unpickler_traverse, /*tp_traverse*/ - (inquiry)Unpickler_clear, /*tp_clear*/ - 0, /*tp_richcompare*/ - 0, /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - Unpickler_methods, /*tp_methods*/ - 0, /*tp_members*/ - Unpickler_getsets, /*tp_getset*/ - 0, /*tp_base*/ - 0, /*tp_dict*/ - 0, /*tp_descr_get*/ - 0, /*tp_descr_set*/ - 0, /*tp_dictoffset*/ - _pickle_Unpickler___init__, /*tp_init*/ - PyType_GenericAlloc, /*tp_alloc*/ - PyType_GenericNew, /*tp_new*/ - PyObject_GC_Del, /*tp_free*/ - 0, /*tp_is_gc*/ +static PyType_Slot pickle_unpickler_type_slots[] = { + {Py_tp_dealloc, Unpickler_dealloc}, + {Py_tp_doc, (char *)_pickle_Unpickler___init____doc__}, + {Py_tp_traverse, Unpickler_traverse}, + {Py_tp_clear, Unpickler_clear}, + {Py_tp_methods, Unpickler_methods}, + {Py_tp_getset, Unpickler_getsets}, + {0,0} +}; + +static PyType_Spec pickle_unpickler_type_spec = { + .name = "_pickle.Unpickler", + .basicsize = sizeof(UnpicklerObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = pickle_unpickler_type_slots, }; /*[clinic input] @@ -7712,7 +7676,7 @@ _pickle_dump_impl(PyObject *module, PyObject *obj, PyObject *file, PyObject *buffer_callback) /*[clinic end generated code: output=706186dba996490c input=5ed6653da99cd97c]*/ { - PicklerObject *pickler = _Pickler_New(); + PicklerObject *pickler = _Pickler_New(module); if (pickler == NULL) return NULL; @@ -7777,7 +7741,7 @@ _pickle_dumps_impl(PyObject *module, PyObject *obj, PyObject *protocol, /*[clinic end generated code: output=fbab0093a5580fdf input=e543272436c6f987]*/ { PyObject *result; - PicklerObject *pickler = _Pickler_New(); + PicklerObject *pickler = _Pickler_New(module); if (pickler == NULL) return NULL; @@ -7843,7 +7807,7 @@ _pickle_load_impl(PyObject *module, PyObject *file, int fix_imports, /*[clinic end generated code: output=250452d141c23e76 input=46c7c31c92f4f371]*/ { PyObject *result; - UnpicklerObject *unpickler = _Unpickler_New(); + UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) return NULL; @@ -7903,7 +7867,7 @@ _pickle_loads_impl(PyObject *module, PyObject *data, int fix_imports, /*[clinic end generated code: output=82ac1e6b588e6d02 input=b3615540d0535087]*/ { PyObject *result; - UnpicklerObject *unpickler = _Unpickler_New(); + UnpicklerObject *unpickler = _Unpickler_New(module); if (unpickler == NULL) return NULL; @@ -7967,83 +7931,102 @@ pickle_traverse(PyObject *m, visitproc visit, void *arg) Py_VISIT(st->codecs_encode); Py_VISIT(st->getattr); Py_VISIT(st->partial); + Py_VISIT(st->pickler_type); + Py_VISIT(st->unpickler_type); return 0; } -static struct PyModuleDef _picklemodule = { - PyModuleDef_HEAD_INIT, - "_pickle", /* m_name */ - pickle_module_doc, /* m_doc */ - sizeof(PickleState), /* m_size */ - pickle_methods, /* m_methods */ - NULL, /* m_reload */ - pickle_traverse, /* m_traverse */ - pickle_clear, /* m_clear */ - (freefunc)pickle_free /* m_free */ -}; - -PyMODINIT_FUNC -PyInit__pickle(void) +static int +_pickle_exec(PyObject *m) { - PyObject *m; PickleState *st; - m = PyState_FindModule(&_picklemodule); - if (m) { - Py_INCREF(m); - return m; - } - if (PyType_Ready(&Pdata_Type) < 0) - return NULL; + return -1; if (PyType_Ready(&PicklerMemoProxyType) < 0) - return NULL; + return -1; if (PyType_Ready(&UnpicklerMemoProxyType) < 0) - return NULL; + return -1; - /* Create the module and add the functions. */ - m = PyModule_Create(&_picklemodule); - if (m == NULL) - return NULL; + st = _Pickle_GetState(m); /* Add types */ - if (PyModule_AddType(m, &Pickler_Type) < 0) { - return NULL; + if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { + return -1; } - if (PyModule_AddType(m, &Unpickler_Type) < 0) { - return NULL; + + st->pickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_pickler_type_spec, NULL); + if (st->pickler_type == NULL) { + return -1; } - if (PyModule_AddType(m, &PyPickleBuffer_Type) < 0) { - return NULL; + + if (PyModule_AddType(m, st->pickler_type) < 0) { + return -1; } - st = _Pickle_GetState(m); + st->unpickler_type = (PyTypeObject *)PyType_FromModuleAndSpec(m, &pickle_unpickler_type_spec, NULL); + if (st->unpickler_type == NULL) { + return -1; + } + + if (PyModule_AddType(m, st->unpickler_type) < 0) { + return -1; + } /* Initialize the exceptions. */ st->PickleError = PyErr_NewException("_pickle.PickleError", NULL, NULL); if (st->PickleError == NULL) - return NULL; + return -1; st->PicklingError = \ PyErr_NewException("_pickle.PicklingError", st->PickleError, NULL); if (st->PicklingError == NULL) - return NULL; + return -1; st->UnpicklingError = \ PyErr_NewException("_pickle.UnpicklingError", st->PickleError, NULL); if (st->UnpicklingError == NULL) - return NULL; + return -1; Py_INCREF(st->PickleError); - if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) - return NULL; + if (PyModule_AddObject(m, "PickleError", st->PickleError) < 0) { + Py_DECREF((PyObject *)st->PickleError); + return -1; + } Py_INCREF(st->PicklingError); - if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) - return NULL; + if (PyModule_AddObject(m, "PicklingError", st->PicklingError) < 0) { + Py_DECREF((PyObject *)st->PicklingError); + return -1; + } Py_INCREF(st->UnpicklingError); - if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) - return NULL; + if (PyModule_AddObject(m, "UnpicklingError", st->UnpicklingError) < 0) { + Py_DECREF((PyObject *)st->UnpicklingError); + return -1; + } if (_Pickle_InitState(st) < 0) - return NULL; + return -1; + + return 0; +} - return m; +static PyModuleDef_Slot pickle_slots[] = { + {Py_mod_exec, _pickle_exec}, + {0, NULL} +}; + +static struct PyModuleDef _picklemodule = { + PyModuleDef_HEAD_INIT, + .m_name = "_pickle", + .m_doc = pickle_module_doc, + .m_size = sizeof(PickleState), + .m_methods = pickle_methods, + .m_slots = pickle_slots, + .m_traverse = pickle_traverse, + .m_clear = pickle_clear, + .m_free = (freefunc)pickle_free +}; + +PyMODINIT_FUNC +PyInit__pickle(void) +{ + return PyModuleDef_Init(&_picklemodule); }