merge 3.3-slp (Stackless python#111)

Author: Anselm Kruis

Include/genobject.h

@@ -33,9 +33,6 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 #define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
 
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
-#ifdef STACKLESS
-#define PyGen_New PyGenerator_New
-#endif
 PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
 PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
 PyObject *_PyGen_Send(PyGenObject *, PyObject *);

Objects/genobject.c

@@ -518,13 +518,8 @@ PyTypeObject PyGen_Type = {
 
 STACKLESS_DECLARE_METHOD(&PyGen_Type, tp_iternext);
 
-#ifdef STACKLESS
-PyObject *
-PyGenerator_New(PyFrameObject *f)
-#else
 PyObject *
 PyGen_New(PyFrameObject *f)
-#endif
 {
     PyGenObject *gen = PyObject_GC_New(PyGenObject, &PyGen_Type);
     if (gen == NULL) {
@@ -533,19 +528,8 @@ PyGen_New(PyFrameObject *f)
     }
     gen->gi_frame = f;
     f->f_gen = (PyObject *) gen;
-#ifdef STACKLESS
-    /* Support for unpickling generators.  This will segmentation fault if
-       called by pricklepit.c:gen_new as that passes Py_None as a placeholder. */
-    if ((PyObject*)f == Py_None) {
-        Py_INCREF(Py_None);
-        gen->gi_code = Py_None;
-    } else {
-#endif
     Py_INCREF(f->f_code);
     gen->gi_code = (PyObject *)(f->f_code);
-#ifdef STACKLESS
-    }
-#endif
     gen->gi_running = 0;
     gen->gi_weakreflist = NULL;
     _PyObject_GC_TRACK(gen);

Stackless/changelog.txt

@@ -9,6 +9,12 @@ What's New in Stackless 3.X.X?
 
 *Release date: 20XX-XX-XX*
 
+- https://bitbucket.org/stackless-dev/stackless/issues/111
+  Restore the Python ABI function PyGen_New(). Previously Stackless named this
+  function PyGenerator_New() and used a macro the redefine PyGen_New as
+  PyGenerator_New. Stackless is now again binary compatible with Python-
+  extensions using PyGen_New().
+
 - https://bitbucket.org/stackless-dev/stackless/issues/107
   Improve unpickling of traceback objects. Stackless now reconstructs the
   frame.f_back linkage in frames directly referenced by traceback objects.

Stackless/core/stackless_impl.h

@@ -128,11 +128,7 @@ struct _frame * slp_clone_frame(struct _frame *f);
 struct _frame * slp_ensure_new_frame(struct _frame *f);
 
 /* exposing some hidden types */
-
-PyAPI_DATA(PyTypeObject) PyGen_Type;
-PyAPI_FUNC(PyObject *) PyGenerator_New(struct _frame *f);
 PyObject * slp_gen_send_ex(PyGenObject *gen, PyObject *arg, int exc);
-#define PyGenerator_Check(op) PyObject_TypeCheck(op, &PyGen_Type)
 
 PyAPI_DATA(PyTypeObject) PyMethodDescr_Type;
 PyAPI_DATA(PyTypeObject) PyClassMethodDescr_Type;

Stackless/pickling/prickelpit.c

@@ -2232,29 +2232,12 @@ static int init_methodwrappertype(void)
 
  ******************************************************/
 
-/* Note: this definition is not compatible to 2.2.3, since
-   the gi_weakreflist does not exist. Therefore, we don't
-   create the object ourselves, but use the provided function
-   with a fake frame. The gi_weakreflist is not touched.
- */
-
-typedef struct {
-    PyObject_HEAD
-    /* The gi_ prefix is intended to remind of generator-iterator. */
-
-    PyFrameObject *gi_frame;
-
-    /* True if generator is being executed. */
-    char gi_running;
-
-    /* List of weak reference. */
-    PyObject *gi_weakreflist;
-} genobject;
-
 static PyTypeObject wrap_PyGen_Type;
+/* Used to initialize a generator created by gen_new. */
+static PyFrameObject *gen_exhausted_frame;
 
 static PyObject *
-gen_reduce(genobject *gen)
+gen_reduce(PyGenObject *gen)
 {
     PyObject *tup;
     tup = Py_BuildValue("(O()(Oi))",
@@ -2268,11 +2251,13 @@ gen_reduce(genobject *gen)
 static PyObject *
 gen_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 {
-    genobject *gen;
-
+    PyGenObject *gen;
     if (is_wrong_type(type)) return NULL;
-    Py_INCREF(Py_None);
-    gen = (genobject *) PyGenerator_New((PyFrameObject *) Py_None);
+
+    /* A reference to frame is stolen by PyGen_New. */
+    assert(gen_exhausted_frame != NULL);
+    assert(PyFrame_Check(gen_exhausted_frame));
+    gen = (PyGenObject *) PyGen_New(slp_ensure_new_frame(gen_exhausted_frame));
     if (gen == NULL)
         return NULL;
     Py_TYPE(gen) = type;
@@ -2282,7 +2267,7 @@ gen_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 static PyObject *
 gen_setstate(PyObject *self, PyObject *args)
 {
-    genobject *gen = (genobject *) self;
+    PyGenObject *gen = (PyGenObject *) self;
     PyFrameObject *f;
     int gi_running;
 
@@ -2294,16 +2279,17 @@ gen_setstate(PyObject *self, PyObject *args)
     if (!gi_running) {
         if ((f = slp_ensure_new_frame(f)) != NULL) {
             /* use a second one for late initialization */
-            genobject *tmpgen;
+            PyGenObject *tmpgen;
             /* PyGenerator_New eats an existing reference */
-            if ((tmpgen = (genobject *)
-                       PyGenerator_New(f)) == NULL) {
+            if ((tmpgen = (PyGenObject *)
+                       PyGen_New(f)) == NULL) {
                 Py_DECREF(f);
                 return NULL;
             }
             Py_INCREF(f);
-            Py_CLEAR(gen->gi_frame);
-            gen->gi_frame = f;
+            Py_SETREF(gen->gi_frame, f);
+            Py_INCREF(f->f_code);
+            Py_SETREF(gen->gi_code, (PyObject *)f->f_code);
             /* The frame the temporary generator references
                will have GeneratorExit raised on it, when the
                temporary generator is torn down.  So clearing
@@ -2327,8 +2313,9 @@ gen_setstate(PyObject *self, PyObject *args)
      * generator without the corresponding tasklet.
      */
     Py_INCREF(f);
-    Py_CLEAR(gen->gi_frame);
-    gen->gi_frame = f;
+    Py_SETREF(gen->gi_frame, f);
+    Py_INCREF(f->f_code);
+    Py_SETREF(gen->gi_code, (PyObject *)f->f_code);
     gen->gi_running = gi_running;
     Py_TYPE(gen) = Py_TYPE(gen)->tp_base;
     Py_INCREF(gen);
@@ -2343,10 +2330,10 @@ DEF_INVALID_EXEC(gen_iternext_callback)
 static int init_generatortype(void)
 {
     int res;
-    genobject *gen = (genobject *) run_script(
-        "def f(): yield 42\n"           /* define a generator */
-        "g = f()\n"                     /* instanciate it */
-        "g.__next__()\n", "g");        /* force callback frame creation */
+    PyGenObject *gen = (PyGenObject *) run_script(
+        "def exhausted_generator(): yield 42\n"   /* define a generator */
+        "g = exhausted_generator()\n"             /* instanciate it */
+        "g.__next__()\n", "g");                       /* force callback frame creation */
     PyFrameObject *cbframe;
 
     if (gen == NULL || gen->gi_frame->f_back == NULL)
@@ -2356,6 +2343,13 @@ static int init_generatortype(void)
               gen->gi_frame->f_back->f_execute,
               REF_INVALID_EXEC(gen_iternext_callback))
           || init_type(&wrap_PyGen_Type, initchain);
+
+    assert(gen_exhausted_frame == NULL);
+    gen_exhausted_frame = slp_ensure_new_frame(gen->gi_frame);
+    if (gen_exhausted_frame == NULL) {
+        return -1;
+    }
+
     Py_DECREF(gen);
     return res;
 }
@@ -2458,23 +2452,36 @@ slp_pickle_moduledict(PyObject *self, PyObject *args)
   source module initialization
 
  ******************************************************/
+static int
+_wrapmodule_traverse(PyObject *self, visitproc visit, void *arg)
+{
+    Py_VISIT(gen_exhausted_frame);
+    return 0;
+}
+
+static int
+_wrapmodule_clear(PyObject *self)
+{
+    Py_CLEAR(gen_exhausted_frame);
+    return 0;
+}
+
 static struct PyModuleDef _wrapmodule = {
     PyModuleDef_HEAD_INIT,
     "_stackless._wrap",
     NULL,
     -1,
     NULL,
     NULL,
-    NULL,
-    NULL,
+    _wrapmodule_traverse,
+    _wrapmodule_clear,
     NULL
 };
 
 PyObject*
 init_prickelpit(void)
 {
     PyObject *copy_reg, *tmp;
-    int ret = 0;
 
     types_mod = PyModule_Create(&_wrapmodule);
     if (types_mod == NULL)
@@ -2491,7 +2498,11 @@ init_prickelpit(void)
         Py_CLEAR(types_mod);
         return NULL;
     }
-    ret = initchain();
+    if (initchain()) {
+        Py_CLEAR(pickle_reg);
+        Py_CLEAR(types_mod);
+        return NULL;
+    }
     Py_CLEAR(pickle_reg);
     tmp = types_mod;
     types_mod = NULL;

Stackless/unittests/test_generator.py

@@ -1,6 +1,7 @@
 import unittest
 import gc
 import stackless
+import types
 
 from support import StacklessTestCase
 
@@ -31,5 +32,22 @@ def testSimpleLeakage(self):
         if len(leakage):
             self.failUnless(len(leakage) == 0, "Leaked %s" % repr(leakage))
 
+
+class TestGeneratorWrapper(StacklessTestCase):
+    def test_run_wrap_generator(self):
+        g = stackless._wrap.generator()
+        self.assertIsInstance(g, types.GeneratorType)
+        self.assertIsNot(type(g), types.GeneratorType)
+        self.assertRaises(StopIteration, next, g)
+
+    def test_wrap_generator_frame_code(self):
+        g0 = stackless._wrap.generator()
+        g1 = stackless._wrap.generator()
+        self.assertIsInstance(g0.gi_frame, types.FrameType)
+        self.assertIsInstance(g0.gi_code, types.CodeType)
+        self.assertIs(g0.gi_code, g1.gi_code)
+        self.assertIsNot(g0.gi_frame, g1.gi_frame)
+        self.assertEqual(g0.__name__, "exhausted_generator")
+
 if __name__ == '__main__':
     unittest.main()