From f64d609b7ed0c68fbaf4106997a11e4d2d83064a Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sat, 20 Jun 2020 22:07:33 +0300 Subject: [PATCH 1/5] bpo-40939: Rename PyPegen* functions to PyParser* Renaame PyPegen* functions to PyParser*, so that we can remove the old set of PyParser* functions that were using the old parser. --- Include/Python.h | 1 + Include/{internal => }/pegen_interface.h | 29 ++++++++------ Include/pythonrun.h | 50 ------------------------ Makefile.pre.in | 2 +- Parser/peg_api.c | 27 ++++++++++--- Python/pythonrun.c | 16 ++++---- 6 files changed, 49 insertions(+), 76 deletions(-) rename Include/{internal => }/pegen_interface.h (57%) diff --git a/Include/Python.h b/Include/Python.h index dcd0a57ac1f03f..3e071ddedf84b9 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -141,6 +141,7 @@ #include "modsupport.h" #include "compile.h" #include "pythonrun.h" +#include "pegen_interface.h" #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" diff --git a/Include/internal/pegen_interface.h b/Include/pegen_interface.h similarity index 57% rename from Include/internal/pegen_interface.h rename to Include/pegen_interface.h index ee4c77ec006760..baccf79f5e83e2 100644 --- a/Include/internal/pegen_interface.h +++ b/Include/pegen_interface.h @@ -4,43 +4,50 @@ extern "C" { #endif -#ifndef Py_BUILD_CORE -# error "this header requires Py_BUILD_CORE define" -#endif - #include "Python.h" #include "Python-ast.h" -PyAPI_FUNC(mod_ty) PyPegen_ASTFromString( +#ifndef Py_LIMITED_API +PyAPI_FUNC(mod_ty) PyParser_ASTFromString( const char *str, const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena); -PyAPI_FUNC(mod_ty) PyPegen_ASTFromStringObject( +PyAPI_FUNC(mod_ty) PyParser_ASTFromStringObject( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); -PyAPI_FUNC(mod_ty) PyPegen_ASTFromFileObject( +PyAPI_FUNC(mod_ty) PyParser_ASTFromFile( FILE *fp, - PyObject *filename_ob, + const char *filename, + const char* enc, int mode, + const char *ps1, + const char *ps2, + PyCompilerFlags *flags, + int *errcode, + PyArena *arena); +PyAPI_FUNC(mod_ty) PyParser_ASTFromFileObject( + FILE *fp, + PyObject *filename_ob, const char *enc, + int mode, const char *ps1, const char *ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena); -PyAPI_FUNC(mod_ty) PyPegen_ASTFromFilename( +PyAPI_FUNC(mod_ty) PyParser_ASTFromFilename( const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena); - +#endif /* !Py_LIMITED_API */ #ifdef __cplusplus } #endif -#endif /* !Py_PEGENINTERFACE*/ +#endif /* !Py_PEGENINTERFACE */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 46091e09216330..d43734b5a12ff0 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -32,57 +32,7 @@ PyAPI_FUNC(int) PyRun_InteractiveLoopFlags( const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromString( - const char *s, - const char *filename, /* decoded from the filesystem encoding */ - int start, - PyCompilerFlags *flags, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromStringObject( - const char *s, - PyObject *filename, - int start, - PyCompilerFlags *flags, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( - FILE *fp, - const char *filename, /* decoded from the filesystem encoding */ - const char* enc, - int start, - const char *ps1, - const char *ps2, - PyCompilerFlags *flags, - int *errcode, - PyArena *arena); -PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( - FILE *fp, - PyObject *filename, - const char* enc, - int start, - const char *ps1, - const char *ps2, - PyCompilerFlags *flags, - int *errcode, - PyArena *arena); -#endif -#ifndef PyParser_SimpleParseString -#define PyParser_SimpleParseString(S, B) \ - PyParser_SimpleParseStringFlags(S, B, 0) -#define PyParser_SimpleParseFile(FP, S, B) \ - PyParser_SimpleParseFileFlags(FP, S, B, 0) -#endif -PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, - int); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 -PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *, - const char *, - int, int); -#endif -PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *, - int, int); - -#ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *, PyCompilerFlags *); diff --git a/Makefile.pre.in b/Makefile.pre.in index fc6dc434e0a18e..44dc8d720161cc 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -307,7 +307,7 @@ PEGEN_OBJS= \ PEGEN_HEADERS= \ - $(srcdir)/Include/internal/pegen_interface.h \ + $(srcdir)/Include/pegen_interface.h \ $(srcdir)/Parser/pegen.h \ $(srcdir)/Parser/string_parser.h diff --git a/Parser/peg_api.c b/Parser/peg_api.c index b947c780765459..3a2aafbcea18d1 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -4,20 +4,20 @@ #include "pegen.h" mod_ty -PyPegen_ASTFromString(const char *str, const char *filename, int mode, +PyParser_ASTFromString(const char *str, const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena) { PyObject *filename_ob = PyUnicode_FromString(filename); if (filename_ob == NULL) { return NULL; } - mod_ty result = PyPegen_ASTFromStringObject(str, filename_ob, mode, flags, arena); + mod_ty result = PyParser_ASTFromStringObject(str, filename_ob, mode, flags, arena); Py_XDECREF(filename_ob); return result; } mod_ty -PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, +PyParser_ASTFromStringObject(const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena) { if (PySys_Audit("compile", "yO", str, filename) < 0) { @@ -29,7 +29,7 @@ PyPegen_ASTFromStringObject(const char *str, PyObject* filename, int mode, } mod_ty -PyPegen_ASTFromFilename(const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena) +PyParser_ASTFromFilename(const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena) { PyObject *filename_ob = PyUnicode_FromString(filename); if (filename_ob == NULL) { @@ -42,8 +42,23 @@ PyPegen_ASTFromFilename(const char *filename, int mode, PyCompilerFlags *flags, } mod_ty -PyPegen_ASTFromFileObject(FILE *fp, PyObject *filename_ob, int mode, - const char *enc, const char *ps1, const char* ps2, +PyParser_ASTFromFile(FILE *fp, const char *filename, const char *enc, + int mode, const char *ps1, const char* ps2, + PyCompilerFlags *flags, int *errcode, PyArena *arena) +{ + PyObject *filename_ob = PyUnicode_FromString(filename); + if (filename_ob == NULL) { + return NULL; + } + mod_ty result = PyParser_ASTFromFileObject(fp, filename_ob, enc, mode, + ps1, ps2, flags, errcode, arena); + Py_XDECREF(filename_ob); + return result; +} + +mod_ty +PyParser_ASTFromFileObject(FILE *fp, PyObject *filename_ob, const char *enc, + int mode, const char *ps1, const char* ps2, PyCompilerFlags *flags, int *errcode, PyArena *arena) { if (PySys_Audit("compile", "OO", Py_None, filename_ob) < 0) { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 04fad04227df9d..f79237635f6b28 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -26,7 +26,7 @@ #include "symtable.h" // PySymtable_BuildObject() #include "marshal.h" // PyMarshal_ReadLongFromFile() -#include "pegen_interface.h" // PyPegen_ASTFrom* +#include "pegen_interface.h" // PyParser_ASTFrom* #ifdef MS_WINDOWS # include "malloc.h" // alloca() @@ -205,8 +205,8 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, return -1; } - mod = PyPegen_ASTFromFileObject(fp, filename, Py_single_input, - enc, ps1, ps2, flags, &errcode, arena); + mod = PyParser_ASTFromFileObject(fp, filename, enc, Py_single_input, + ps1, ps2, flags, &errcode, arena); Py_XDECREF(v); Py_XDECREF(w); @@ -1026,7 +1026,7 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals, if (arena == NULL) return NULL; - mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); if (mod != NULL) ret = run_mod(mod, filename, globals, locals, flags, arena); @@ -1051,8 +1051,8 @@ PyRun_FileExFlags(FILE *fp, const char *filename_str, int start, PyObject *globa if (arena == NULL) goto exit; - mod = PyPegen_ASTFromFileObject(fp, filename, start, NULL, NULL, NULL, - flags, NULL, arena); + mod = PyParser_ASTFromFileObject(fp, filename, NULL, start, NULL, NULL, + flags, NULL, arena); if (closeit) fclose(fp); @@ -1200,7 +1200,7 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, if (arena == NULL) return NULL; - mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); if (mod == NULL) { PyArena_Free(arena); return NULL; @@ -1303,7 +1303,7 @@ _Py_SymtableStringObjectFlags(const char *str, PyObject *filename, int start, Py if (arena == NULL) return NULL; - mod = PyPegen_ASTFromStringObject(str, filename, start, flags, arena); + mod = PyParser_ASTFromStringObject(str, filename, start, flags, arena); if (mod == NULL) { PyArena_Free(arena); return NULL; From e41aa08a1b5cfa9405ec2923b5dc13875287a1d4 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 20 Jun 2020 19:27:51 +0000 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20blu?= =?UTF-8?q?rb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Core and Builtins/2020-06-20-19-27-47.bpo-40939.jxJ4yn.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-06-20-19-27-47.bpo-40939.jxJ4yn.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-20-19-27-47.bpo-40939.jxJ4yn.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-20-19-27-47.bpo-40939.jxJ4yn.rst new file mode 100644 index 00000000000000..7024dfe47ae1c7 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-20-19-27-47.bpo-40939.jxJ4yn.rst @@ -0,0 +1 @@ +Rename `PyPegen*` functions to `PyParser*`, so that we can remove the old set of `PyParser*` functions that were using the old parser, but keep everything backwards-compatible. \ No newline at end of file From 27704337ec9b0f7c9714e6dd8c6889c1860e82a7 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sun, 21 Jun 2020 02:06:07 +0300 Subject: [PATCH 3/5] Fix windows build --- PCbuild/pythoncore.vcxproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 0f9110e08d65b2..4acfe436e19fd3 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -162,7 +162,6 @@ - @@ -213,6 +212,7 @@ + From 867fb8c2f62a87e3b10c28b5557266a8c737587c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sun, 21 Jun 2020 21:49:18 +0300 Subject: [PATCH 4/5] Remove Python-ast.h import in pegen_interface.h --- Include/pegen_interface.h | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Include/pegen_interface.h b/Include/pegen_interface.h index baccf79f5e83e2..1c6576d926d8d3 100644 --- a/Include/pegen_interface.h +++ b/Include/pegen_interface.h @@ -5,22 +5,21 @@ extern "C" { #endif #include "Python.h" -#include "Python-ast.h" #ifndef Py_LIMITED_API -PyAPI_FUNC(mod_ty) PyParser_ASTFromString( +PyAPI_FUNC(struct _mod *) PyParser_ASTFromString( const char *str, const char *filename, int mode, PyCompilerFlags *flags, PyArena *arena); -PyAPI_FUNC(mod_ty) PyParser_ASTFromStringObject( +PyAPI_FUNC(struct _mod *) PyParser_ASTFromStringObject( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); -PyAPI_FUNC(mod_ty) PyParser_ASTFromFile( +PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile( FILE *fp, const char *filename, const char* enc, @@ -30,7 +29,7 @@ PyAPI_FUNC(mod_ty) PyParser_ASTFromFile( PyCompilerFlags *flags, int *errcode, PyArena *arena); -PyAPI_FUNC(mod_ty) PyParser_ASTFromFileObject( +PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( FILE *fp, PyObject *filename_ob, const char *enc, @@ -40,7 +39,7 @@ PyAPI_FUNC(mod_ty) PyParser_ASTFromFileObject( PyCompilerFlags *flags, int *errcode, PyArena *arena); -PyAPI_FUNC(mod_ty) PyParser_ASTFromFilename( +PyAPI_FUNC(struct _mod *) PyParser_ASTFromFilename( const char *filename, int mode, PyCompilerFlags *flags, From fd6bbe2a9a20eded451e0d2119e0cd294b8cca86 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Sun, 21 Jun 2020 23:37:42 +0300 Subject: [PATCH 5/5] Rename pegen_interface.h to parser_interface.h --- Include/Python.h | 2 +- Include/{pegen_interface.h => parser_interface.h} | 0 Makefile.pre.in | 2 +- PCbuild/pythoncore.vcxproj | 2 +- Parser/peg_api.c | 2 +- Python/pythonrun.c | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename Include/{pegen_interface.h => parser_interface.h} (100%) diff --git a/Include/Python.h b/Include/Python.h index 3e071ddedf84b9..57f71d41d8d477 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -141,7 +141,7 @@ #include "modsupport.h" #include "compile.h" #include "pythonrun.h" -#include "pegen_interface.h" +#include "parser_interface.h" #include "pylifecycle.h" #include "ceval.h" #include "sysmodule.h" diff --git a/Include/pegen_interface.h b/Include/parser_interface.h similarity index 100% rename from Include/pegen_interface.h rename to Include/parser_interface.h diff --git a/Makefile.pre.in b/Makefile.pre.in index 44dc8d720161cc..24dddcf56f8fa3 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -307,7 +307,7 @@ PEGEN_OBJS= \ PEGEN_HEADERS= \ - $(srcdir)/Include/pegen_interface.h \ + $(srcdir)/Include/parser_interface.h \ $(srcdir)/Parser/pegen.h \ $(srcdir)/Parser/string_parser.h diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 4acfe436e19fd3..fc99d7748a01fc 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -212,7 +212,7 @@ - + diff --git a/Parser/peg_api.c b/Parser/peg_api.c index 3a2aafbcea18d1..8381d5e86b0db5 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -1,4 +1,4 @@ -#include "pegen_interface.h" +#include "parser_interface.h" #include "tokenizer.h" #include "pegen.h" diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f79237635f6b28..ff80103050e4e2 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -26,7 +26,7 @@ #include "symtable.h" // PySymtable_BuildObject() #include "marshal.h" // PyMarshal_ReadLongFromFile() -#include "pegen_interface.h" // PyParser_ASTFrom* +#include "parser_interface.h" // PyParser_ASTFrom* #ifdef MS_WINDOWS # include "malloc.h" // alloca()