From d5f14fcc97cc3ba37e0b1a327413bffecc9ced05 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 4 Nov 2022 16:50:19 +0000 Subject: [PATCH 01/13] extract compiler_codegen from compiler_mod --- Python/compile.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 065d1b08d0642f..37dd174689002e 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -2145,15 +2145,13 @@ compiler_body(struct compiler *c, location loc, asdl_stmt_seq *stmts) return 1; } -static PyCodeObject * -compiler_mod(struct compiler *c, mod_ty mod) +static int +compiler_codegen(struct compiler *c, mod_ty mod) { - PyCodeObject *co; - int addNone = 1; _Py_DECLARE_STR(anon_module, ""); if (!compiler_enter_scope(c, &_Py_STR(anon_module), COMPILER_SCOPE_MODULE, mod, 1)) { - return NULL; + return 0; } location loc = LOCATION(1, 1, 0, 0); switch (mod->kind) { @@ -2172,7 +2170,6 @@ compiler_mod(struct compiler *c, mod_ty mod) break; case Expression_kind: VISIT_IN_SCOPE(c, expr, mod->v.Expression.body); - addNone = 0; break; default: PyErr_Format(PyExc_SystemError, @@ -2180,7 +2177,17 @@ compiler_mod(struct compiler *c, mod_ty mod) mod->kind); return 0; } - co = assemble(c, addNone); + return 1; +} + +static PyCodeObject * +compiler_mod(struct compiler *c, mod_ty mod) +{ + int addNone = mod->kind != Expression_kind; + if (!compiler_codegen(c, mod)) { + return NULL; + } + PyCodeObject *co = assemble(c, addNone); compiler_exit_scope(c); return co; } From 89b8357f3c075dbab4ca1012b21ec362110e1ce1 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 4 Nov 2022 21:59:22 +0000 Subject: [PATCH 02/13] add unit tests for codegen --- Include/internal/pycore_compile.h | 7 ++ Include/internal/pycore_global_strings.h | 1 + .../internal/pycore_runtime_init_generated.h | 7 ++ Lib/test/support/bytecode_helper.py | 91 ++++++++------ Lib/test/test_compiler_codegen.py | 51 ++++++++ Modules/_testinternalcapi.c | 23 +++- Modules/clinic/_testinternalcapi.c.h | 65 +++++++++- Python/compile.c | 114 ++++++++++++++---- 8 files changed, 294 insertions(+), 65 deletions(-) create mode 100644 Lib/test/test_compiler_codegen.py diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index cb490abe77a59d..967fe92a5bc2b2 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -40,6 +40,13 @@ extern int _PyAST_Optimize( _PyASTOptimizeState *state); /* Access compiler internals for unit testing */ + +PyAPI_FUNC(PyObject*) _PyCompile_CodeGen( + PyObject *ast, + PyObject *filename, + PyCompilerFlags *flags, + int optimize); + PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg( PyObject *instructions, PyObject *consts); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 002d81e3e7db66..e08680b6ad3069 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -265,6 +265,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(argdefs) STRUCT_FOR_ID(arguments) STRUCT_FOR_ID(argv) + STRUCT_FOR_ID(ast) STRUCT_FOR_ID(attribute) STRUCT_FOR_ID(authorizer_callback) STRUCT_FOR_ID(b) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 88b84cbe0166a2..871e26b32def2d 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -774,6 +774,7 @@ extern "C" { INIT_ID(argdefs), \ INIT_ID(arguments), \ INIT_ID(argv), \ + INIT_ID(ast), \ INIT_ID(attribute), \ INIT_ID(authorizer_callback), \ INIT_ID(b), \ @@ -1896,6 +1897,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(argv); PyUnicode_InternInPlace(&string); + string = &_Py_ID(ast); + PyUnicode_InternInPlace(&string); string = &_Py_ID(attribute); PyUnicode_InternInPlace(&string); string = &_Py_ID(authorizer_callback); @@ -5765,6 +5768,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(argv)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(ast)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(ast)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(attribute)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(attribute)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index eb4ae1a68e3fc1..c553048e1908f6 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,7 +3,7 @@ import unittest import dis import io -from _testinternalcapi import optimize_cfg +from _testinternalcapi import compiler_codegen, optimize_cfg _UNSPECIFIED = object() @@ -44,8 +44,7 @@ def assertNotInBytecode(self, x, opname, argval=_UNSPECIFIED): msg = msg % (opname, argval, disassembly) self.fail(msg) - -class CfgOptimizationTestCase(unittest.TestCase): +class CompilationStepTestCase(unittest.TestCase): HAS_ARG = set(dis.hasarg) HAS_TARGET = set(dis.hasjrel + dis.hasjabs + dis.hasexc) @@ -58,24 +57,35 @@ def Label(self): self.last_label += 1 return self.last_label - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - instructions = [] - for item in insts: - if isinstance(item, int): - instructions.append(item) - else: - assert isinstance(item, tuple) - inst = list(reversed(item)) - opcode = dis.opmap[inst.pop()] - oparg = inst.pop() if opcode in self.HAS_ARG_OR_TARGET else 0 - loc = inst + [-1] * (4 - len(inst)) - instructions.append((opcode, oparg, *loc)) - return instructions + def compareInstructions(self, actual_, expected_): + # get two lists where each entry is a label or + # an instruction tuple. Compare them, while mapping + # each actual label to a corresponding expected label + # based on their locations. + + self.assertIsInstance(actual_, list) + self.assertIsInstance(expected_, list) + + actual = self.normalize_insts(actual_) + expected = self.normalize_insts(expected_) + self.assertEqual(len(actual), len(expected)) + + # compare instructions + for act, exp in zip(actual, expected): + if isinstance(act, int): + self.assertEqual(exp, act) + continue + self.assertIsInstance(exp, tuple) + self.assertIsInstance(act, tuple) + # crop comparison to the provided expected values + if len(act) > len(exp): + act = act[:len(exp)] + self.assertEqual(exp, act) def normalize_insts(self, insts): """ Map labels to instruction index. Remove labels which are not used as jump targets. + Map opcodes to opnames. """ labels_map = {} targets = set() @@ -107,31 +117,34 @@ def normalize_insts(self, insts): res.append((opcode, arg, *loc)) return res + +class CodegenTestCase(CompilationStepTestCase): + + def generate_code(self, ast): + insts = compiler_codegen(ast, "my_file.py", 0) + return insts + + +class CfgOptimizationTestCase(CompilationStepTestCase): + + def complete_insts_info(self, insts): + # fill in omitted fields in location, and oparg 0 for ops with no arg. + instructions = [] + for item in insts: + if isinstance(item, int): + instructions.append(item) + else: + assert isinstance(item, tuple) + inst = list(reversed(item)) + opcode = dis.opmap[inst.pop()] + oparg = inst.pop() if opcode in self.HAS_ARG_OR_TARGET else 0 + loc = inst + [-1] * (4 - len(inst)) + instructions.append((opcode, oparg, *loc)) + return instructions + def get_optimized(self, insts, consts): insts = self.complete_insts_info(insts) insts = optimize_cfg(insts, consts) return insts, consts - def compareInstructions(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Compare them, while mapping - # each actual label to a corresponding expected label - # based on their locations. - - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) - self.assertEqual(len(actual), len(expected)) - - # compare instructions - for act, exp in zip(actual, expected): - if isinstance(act, int): - self.assertEqual(exp, act) - continue - self.assertIsInstance(exp, tuple) - self.assertIsInstance(act, tuple) - # pad exp with -1's (if location info is incomplete) - exp += (-1,) * (len(act) - len(exp)) - self.assertEqual(exp, act) diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py new file mode 100644 index 00000000000000..13962eca58daf3 --- /dev/null +++ b/Lib/test/test_compiler_codegen.py @@ -0,0 +1,51 @@ + +from test.support.bytecode_helper import CodegenTestCase + +# Tests for the code-generation stage of the compiler. +# Examine the un-optimized code generated from the AST. + +class IsolatedCodeGenTests(CodegenTestCase): + + def codegen_test(self, snippet, expected_insts): + import ast + a = ast.parse(snippet, "my_file.py", "exec"); + insts = self.generate_code(a) + self.compareInstructions(insts, expected_insts) + + def test_if_expression(self): + snippet = "42 if True else 24" + false_lbl = self.Label() + expected = [ + ('RESUME', 0, 0), + ('LOAD_CONST', 0, 1), + ('POP_JUMP_IF_FALSE', false_lbl := self.Label(), 1), + ('LOAD_CONST', 1, 1), + ('JUMP', exit_lbl := self.Label()), + false_lbl, + ('LOAD_CONST', 2, 1), + exit_lbl, + ('POP_TOP', None), + ] + self.codegen_test(snippet, expected) + + def test_for_loop(self): + snippet = "for x in l:\n\tprint(x)" + false_lbl = self.Label() + expected = [ + ('RESUME', 0, 0), + ('LOAD_NAME', 0, 1), + ('GET_ITER', None, 1), + loop_lbl := self.Label(), + ('FOR_ITER', exit_lbl := self.Label(), 1), + ('STORE_NAME', None, 1), + ('PUSH_NULL', None, 2), + ('LOAD_NAME', None, 2), + ('LOAD_NAME', None, 2), + ('CALL', None, 2), + ('POP_TOP', None), + ('JUMP', loop_lbl), + exit_lbl, + ('END_FOR', None), + ] + self.codegen_test(snippet, expected) + diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index d74f92808d3aba..6987618119dc92 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -14,7 +14,7 @@ #include "Python.h" #include "pycore_atomic_funcs.h" // _Py_atomic_int_get() #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_compile.h" // _PyCompile_OptimizeCfg() +#include "pycore_compile.h" // _PyCompile_CodeGen, _PyCompile_OptimizeCfg #include "pycore_fileutils.h" // _Py_normpath #include "pycore_frame.h" // _PyInterpreterFrame #include "pycore_gc.h" // PyGC_Head @@ -530,6 +530,26 @@ set_eval_frame_record(PyObject *self, PyObject *list) Py_RETURN_NONE; } +/*[clinic input] + +_testinternalcapi.compiler_codegen -> object + + ast: object + filename: object + optimize: int + +Apply compiler code generation to an AST. +[clinic start generated code]*/ + +static PyObject * +_testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, + PyObject *filename, int optimize) +/*[clinic end generated code: output=fbbbbfb34700c804 input=e9fbe6562f7f75e4]*/ +{ + PyCompilerFlags *flags = NULL; + return _PyCompile_CodeGen(ast, filename, flags, optimize); +} + /*[clinic input] @@ -613,6 +633,7 @@ static PyMethodDef TestMethods[] = { {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, {"set_eval_frame_default", set_eval_frame_default, METH_NOARGS, NULL}, {"set_eval_frame_record", set_eval_frame_record, METH_O, NULL}, + _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF {"get_interp_settings", get_interp_settings, METH_VARARGS, NULL}, {NULL, NULL} /* sentinel */ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 8113fff37997af..e8d5681b194916 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -8,6 +8,69 @@ preserve #endif +PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, +"compiler_codegen($module, /, ast, filename, optimize)\n" +"--\n" +"\n" +"Apply compiler code generation to an AST."); + +#define _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF \ + {"compiler_codegen", _PyCFunction_CAST(_testinternalcapi_compiler_codegen), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_compiler_codegen__doc__}, + +static PyObject * +_testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, + PyObject *filename, int optimize); + +static PyObject * +_testinternalcapi_compiler_codegen(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 3 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(ast), &_Py_ID(filename), &_Py_ID(optimize), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"ast", "filename", "optimize", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "compiler_codegen", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[3]; + PyObject *ast; + PyObject *filename; + int optimize; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); + if (!args) { + goto exit; + } + ast = args[0]; + filename = args[1]; + optimize = _PyLong_AsInt(args[2]); + if (optimize == -1 && PyErr_Occurred()) { + goto exit; + } + return_value = _testinternalcapi_compiler_codegen_impl(module, ast, filename, optimize); + +exit: + return return_value; +} + PyDoc_STRVAR(_testinternalcapi_optimize_cfg__doc__, "optimize_cfg($module, /, instructions, consts)\n" "--\n" @@ -65,4 +128,4 @@ _testinternalcapi_optimize_cfg(PyObject *module, PyObject *const *args, Py_ssize exit: return return_value; } -/*[clinic end generated code: output=3b1fd713290f68a9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=efe95836482fd542 input=a9049054013a1b77]*/ diff --git a/Python/compile.c b/Python/compile.c index 37dd174689002e..e2a0b3ee5e0465 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -606,44 +606,55 @@ compiler_init(struct compiler *c) return 1; } -PyCodeObject * -_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, - int optimize, PyArena *arena) +static int +compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, + PyCompilerFlags *flags, int optimize, PyArena *arena) { - struct compiler c; - PyCodeObject *co = NULL; PyCompilerFlags local_flags = _PyCompilerFlags_INIT; - int merged; - if (!compiler_init(&c)) - return NULL; Py_INCREF(filename); - c.c_filename = filename; - c.c_arena = arena; - if (!_PyFuture_FromAST(mod, filename, &c.c_future)) { - goto finally; + c->c_filename = filename; + c->c_arena = arena; + if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { + return 0; } if (!flags) { flags = &local_flags; } - merged = c.c_future.ff_features | flags->cf_flags; - c.c_future.ff_features = merged; + int merged = c->c_future.ff_features | flags->cf_flags; + c->c_future.ff_features = merged; flags->cf_flags = merged; - c.c_flags = flags; - c.c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; - c.c_nestlevel = 0; + c->c_flags = flags; + c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; + c->c_nestlevel = 0; _PyASTOptimizeState state; - state.optimize = c.c_optimize; + state.optimize = c->c_optimize; state.ff_features = merged; if (!_PyAST_Optimize(mod, arena, &state)) { - goto finally; + return 0; } - - c.c_st = _PySymtable_Build(mod, filename, &c.c_future); - if (c.c_st == NULL) { - if (!PyErr_Occurred()) + c->c_st = _PySymtable_Build(mod, filename, &c->c_future); + if (c->c_st == NULL) { + if (!PyErr_Occurred()) { PyErr_SetString(PyExc_SystemError, "no symtable"); + } + return 0; + } + return 1; +} + +PyCodeObject * +_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, + int optimize, PyArena *arena) +{ + struct compiler c; + PyCodeObject *co = NULL; + if (!compiler_init(&c)) { + return NULL; + } + + if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { goto finally; } @@ -9831,6 +9842,9 @@ duplicate_exits_without_lineno(cfg_builder *g) /* Access to compiler optimizations for unit tests. + * + * _PyCompile_CodeGen takes and AST, applies code-gen and + * returns the unoptimized CFG as an instruction list. * * _PyCompile_OptimizeCfg takes an instruction list, constructs * a CFG, optimizes it and converts back to an instruction list. @@ -9926,7 +9940,10 @@ cfg_to_instructions(cfg_builder *g) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; location loc = instr->i_loc; - int arg = HAS_TARGET(instr->i_opcode) ? instr->i_target->b_label : instr->i_oparg; + int arg = instr->i_oparg; + if (HAS_TARGET(instr->i_opcode) && instr->i_target != NULL) { + arg = instr->i_target->b_label; + } PyObject *inst_tuple = Py_BuildValue( "(iiiiii)", instr->i_opcode, arg, loc.lineno, loc.end_lineno, @@ -9949,6 +9966,55 @@ cfg_to_instructions(cfg_builder *g) return NULL; } +PyObject * +_PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *flags, + int optimize) +{ + PyObject *res = NULL; + + if (!PyAST_Check(ast)) { + PyErr_SetString(PyExc_TypeError, "expected an AST"); + return NULL; + } + + PyArena *arena = _PyArena_New(); + if (arena == NULL) { + return NULL; + } + + mod_ty mod = PyAST_obj2mod(ast, arena, 0 /* exec */); + if (mod == NULL || !_PyAST_Validate(mod)) { + _PyArena_Free(arena); + return NULL; + } + + /* Create and set up the compiler */ + struct compiler c; + if (!compiler_init(&c)) { + _PyArena_Free(arena); + return NULL; + } + + if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { + goto finally; + } + + if (!compiler_codegen(&c, mod)) { + goto finally; + } + + cfg_builder *g = CFG_BUILDER(&c); + + if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { + goto finally; + } + + res = cfg_to_instructions(g); + +finally: + compiler_free(&c); + return res; +} PyObject * _PyCompile_OptimizeCfg(PyObject *instructions, PyObject *consts) From ddac06a7402e6f06d9109044a6c0dc0a0a3dfec3 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 5 Nov 2022 10:11:02 +0000 Subject: [PATCH 03/13] fix whitespace --- Lib/test/support/bytecode_helper.py | 2 -- Lib/test/test_compiler_codegen.py | 1 - 2 files changed, 3 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index c553048e1908f6..d169f94d81adb2 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -146,5 +146,3 @@ def get_optimized(self, insts, consts): insts = self.complete_insts_info(insts) insts = optimize_cfg(insts, consts) return insts, consts - - diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index 13962eca58daf3..b0a8238db07221 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -48,4 +48,3 @@ def test_for_loop(self): ('END_FOR', None), ] self.codegen_test(snippet, expected) - From b07b9c6011e1d1f11ed75168ce1651f86800147a Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Sat, 5 Nov 2022 21:00:22 +0000 Subject: [PATCH 04/13] fix refleak --- Python/compile.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/compile.c b/Python/compile.c index e2a0b3ee5e0465..e83e0c9dacbd82 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -10012,7 +10012,9 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *flags, res = cfg_to_instructions(g); finally: + compiler_exit_scope(&c); compiler_free(&c); + _PyArena_Free(arena); return res; } From 9bafd3a6c5964204abb9d4daa58a460cff5bbbd3 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 8 Nov 2022 20:57:22 +0000 Subject: [PATCH 05/13] fix flags bug --- Python/compile.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index e83e0c9dacbd82..6dd576e7ba3c32 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -610,16 +610,12 @@ static int compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, PyCompilerFlags *flags, int optimize, PyArena *arena) { - PyCompilerFlags local_flags = _PyCompilerFlags_INIT; Py_INCREF(filename); c->c_filename = filename; c->c_arena = arena; if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return 0; } - if (!flags) { - flags = &local_flags; - } int merged = c->c_future.ff_features | flags->cf_flags; c->c_future.ff_features = merged; flags->cf_flags = merged; @@ -649,11 +645,16 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, int optimize, PyArena *arena) { struct compiler c; - PyCodeObject *co = NULL; if (!compiler_init(&c)) { return NULL; } + PyCompilerFlags local_flags = _PyCompilerFlags_INIT; + if (!flags) { + flags = &local_flags; + } + + PyCodeObject *co = NULL; if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { goto finally; } From ce45de82a3a8a4d064d8dec0dea74dd9899da983 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 8 Nov 2022 22:08:28 +0000 Subject: [PATCH 06/13] regenerate files --- Include/internal/pycore_global_objects_fini_generated.h | 4 ++++ Include/internal/pycore_unicodeobject_generated.h | 2 ++ 2 files changed, 6 insertions(+) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index fad15f631476cf..865a5930391dc9 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -3030,6 +3030,10 @@ _PyStaticObjects_CheckRefcnt(void) { _PyObject_Dump((PyObject *)&_Py_ID(as_integer_ratio)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); }; + if (Py_REFCNT((PyObject *)&_Py_ID(ast)) < _PyObject_IMMORTAL_REFCNT) { + _PyObject_Dump((PyObject *)&_Py_ID(ast)); + Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); + }; if (Py_REFCNT((PyObject *)&_Py_ID(attribute)) < _PyObject_IMMORTAL_REFCNT) { _PyObject_Dump((PyObject *)&_Py_ID(attribute)); Py_FatalError("immortal object has less refcnt than expected _PyObject_IMMORTAL_REFCNT"); diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 026945f1e342a9..e42a4b089a7257 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -440,6 +440,8 @@ _PyUnicode_InitStaticStrings(void) { PyUnicode_InternInPlace(&string); string = &_Py_ID(as_integer_ratio); PyUnicode_InternInPlace(&string); + string = &_Py_ID(ast); + PyUnicode_InternInPlace(&string); string = &_Py_ID(attribute); PyUnicode_InternInPlace(&string); string = &_Py_ID(authorizer_callback); From d98f1503c59d2571a61cfd747d24479cc32b7583 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Tue, 8 Nov 2022 22:37:00 +0000 Subject: [PATCH 07/13] fix flags in _PyCompile_CodeGen --- Python/compile.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/compile.c b/Python/compile.c index 6dd576e7ba3c32..9596e7e7adca20 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -9996,6 +9996,10 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *flags, return NULL; } + PyCompilerFlags local_flags = _PyCompilerFlags_INIT; + if (!flags) { + flags = &local_flags; + } if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { goto finally; } From 00aea64608a8d34c1cc5109459a5893a7d0867e5 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Wed, 9 Nov 2022 10:40:53 +0000 Subject: [PATCH 08/13] PyCompilerFlags stored on struct compiler by value --- Python/compile.c | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 9596e7e7adca20..9d729b2ce9b719 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -122,7 +122,7 @@ (opcode) == STORE_FAST__STORE_FAST) #define IS_TOP_LEVEL_AWAIT(c) ( \ - (c->c_flags->cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ + (c->c_flags.cf_flags & PyCF_ALLOW_TOP_LEVEL_AWAIT) \ && (c->u->u_ste->ste_type == ModuleBlock)) typedef _PyCompilerSrcLocation location; @@ -418,7 +418,7 @@ struct compiler { PyObject *c_filename; struct symtable *c_st; PyFutureFeatures c_future; /* module's __future__ */ - PyCompilerFlags *c_flags; + PyCompilerFlags c_flags; int c_optimize; /* optimization level */ int c_interactive; /* true if in interactive mode */ @@ -608,7 +608,7 @@ compiler_init(struct compiler *c) static int compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags *flags, int optimize, PyArena *arena) + PyCompilerFlags flags, int optimize, PyArena *arena) { Py_INCREF(filename); c->c_filename = filename; @@ -616,9 +616,9 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { return 0; } - int merged = c->c_future.ff_features | flags->cf_flags; + int merged = c->c_future.ff_features | flags.cf_flags; c->c_future.ff_features = merged; - flags->cf_flags = merged; + flags.cf_flags = merged; c->c_flags = flags; c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_nestlevel = 0; @@ -641,7 +641,7 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, } PyCodeObject * -_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, +_PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int optimize, PyArena *arena) { struct compiler c; @@ -649,10 +649,7 @@ _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, return NULL; } - PyCompilerFlags local_flags = _PyCompilerFlags_INIT; - if (!flags) { - flags = &local_flags; - } + PyCompilerFlags flags = pflags ? *pflags : _PyCompilerFlags_INIT; PyCodeObject *co = NULL; if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { @@ -8284,7 +8281,7 @@ compute_code_flags(struct compiler *c) } /* (Only) inherit compilerflags in PyCF_MASK */ - flags |= (c->c_flags->cf_flags & PyCF_MASK); + flags |= (c->c_flags.cf_flags & PyCF_MASK); if ((IS_TOP_LEVEL_AWAIT(c)) && ste->ste_coroutine && @@ -9968,7 +9965,7 @@ cfg_to_instructions(cfg_builder *g) } PyObject * -_PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *flags, +_PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, int optimize) { PyObject *res = NULL; @@ -9996,10 +9993,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *flags, return NULL; } - PyCompilerFlags local_flags = _PyCompilerFlags_INIT; - if (!flags) { - flags = &local_flags; - } + PyCompilerFlags flags = pflags ? *pflags : _PyCompilerFlags_INIT; if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { goto finally; } From ab3f67c8426072bbfc93129785e64cccf6d0cbd3 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Thu, 10 Nov 2022 13:04:58 +0000 Subject: [PATCH 09/13] regen --- Include/internal/pycore_global_objects_fini_generated.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 3d8bc7284f13b9..d6169c00a9fb91 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -780,6 +780,7 @@ _PyStaticObjects_CheckRefcnt(void) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(arguments)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(argv)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(as_integer_ratio)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(ast)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(attribute)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(authorizer_callback)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(b)); From 4453129b437418d22766441fc9be2fd270ce00d2 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 11 Nov 2022 13:34:13 +0000 Subject: [PATCH 10/13] compareInstructions --> assertInstructionsMatch --- Lib/test/support/bytecode_helper.py | 2 +- Lib/test/test_compiler_codegen.py | 2 +- Lib/test/test_peepholer.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index d169f94d81adb2..65ae7a227baafe 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -57,7 +57,7 @@ def Label(self): self.last_label += 1 return self.last_label - def compareInstructions(self, actual_, expected_): + def assertInstructionsMatch(self, actual_, expected_): # get two lists where each entry is a label or # an instruction tuple. Compare them, while mapping # each actual label to a corresponding expected label diff --git a/Lib/test/test_compiler_codegen.py b/Lib/test/test_compiler_codegen.py index b0a8238db07221..f2e14c1e628c01 100644 --- a/Lib/test/test_compiler_codegen.py +++ b/Lib/test/test_compiler_codegen.py @@ -10,7 +10,7 @@ def codegen_test(self, snippet, expected_insts): import ast a = ast.parse(snippet, "my_file.py", "exec"); insts = self.generate_code(a) - self.compareInstructions(insts, expected_insts) + self.assertInstructionsMatch(insts, expected_insts) def test_if_expression(self): snippet = "42 if True else 24" diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 0d398fc3030948..239c9d03fd9d1f 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -984,7 +984,7 @@ def cfg_optimization_test(self, insts, expected_insts, if expected_consts is None: expected_consts = consts opt_insts, opt_consts = self.get_optimized(insts, consts) - self.compareInstructions(opt_insts, expected_insts) + self.assertInstructionsMatch(opt_insts, expected_insts) self.assertEqual(opt_consts, expected_consts) def test_conditional_jump_forward_non_const_condition(self): From ad1ef543bdb21982b85ac867f79759a159f6b77e Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 11 Nov 2022 13:59:13 +0000 Subject: [PATCH 11/13] new_compiler --- Python/compile.c | 70 ++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index d29c88816a32fd..30e8c5c4b9722d 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -584,11 +584,11 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) return result; } + static int -compiler_init(struct compiler *c) +compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, + PyCompilerFlags flags, int optimize, PyArena *arena) { - memset(c, 0, sizeof(struct compiler)); - c->c_const_cache = PyDict_New(); if (!c->c_const_cache) { return 0; @@ -596,17 +596,9 @@ compiler_init(struct compiler *c) c->c_stack = PyList_New(0); if (!c->c_stack) { - Py_CLEAR(c->c_const_cache); return 0; } - return 1; -} - -static int -compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, - PyCompilerFlags flags, int optimize, PyArena *arena) -{ c->c_filename = Py_NewRef(filename); c->c_arena = arena; if (!_PyFuture_FromAST(mod, filename, &c->c_future)) { @@ -636,26 +628,33 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, return 1; } +struct compiler* +new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, + int optimize, PyArena *arena) +{ + PyCompilerFlags flags = pflags ? *pflags : _PyCompilerFlags_INIT; + struct compiler *c = PyMem_Calloc(1, sizeof(struct compiler)); + if (c == NULL) { + return NULL; + } + if (!compiler_setup(c, mod, filename, flags, optimize, arena)) { + compiler_free(c); + return NULL; + } + return c; +} + PyCodeObject * _PyAST_Compile(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int optimize, PyArena *arena) { - struct compiler c; - if (!compiler_init(&c)) { + struct compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + if (c == NULL) { return NULL; } - PyCompilerFlags flags = pflags ? *pflags : _PyCompilerFlags_INIT; - - PyCodeObject *co = NULL; - if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { - goto finally; - } - - co = compiler_mod(&c, mod); - - finally: - compiler_free(&c); + PyCodeObject *co = compiler_mod(c, mod); + compiler_free(c); assert(co || PyErr_Occurred()); return co; } @@ -666,8 +665,9 @@ compiler_free(struct compiler *c) if (c->c_st) _PySymtable_Free(c->c_st); Py_XDECREF(c->c_filename); - Py_DECREF(c->c_const_cache); - Py_DECREF(c->c_stack); + Py_XDECREF(c->c_const_cache); + Py_XDECREF(c->c_stack); + PyMem_Free(c); } static PyObject * @@ -9961,23 +9961,17 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, return NULL; } - /* Create and set up the compiler */ - struct compiler c; - if (!compiler_init(&c)) { + struct compiler *c = new_compiler(mod, filename, pflags, optimize, arena); + if (c == NULL) { _PyArena_Free(arena); return NULL; } - PyCompilerFlags flags = pflags ? *pflags : _PyCompilerFlags_INIT; - if (!compiler_setup(&c, mod, filename, flags, optimize, arena)) { - goto finally; - } - - if (!compiler_codegen(&c, mod)) { + if (!compiler_codegen(c, mod)) { goto finally; } - cfg_builder *g = CFG_BUILDER(&c); + cfg_builder *g = CFG_BUILDER(c); if (translate_jump_labels_to_targets(g->g_entryblock) < 0) { goto finally; @@ -9986,8 +9980,8 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, res = cfg_to_instructions(g); finally: - compiler_exit_scope(&c); - compiler_free(&c); + compiler_exit_scope(c); + compiler_free(c); _PyArena_Free(arena); return res; } From 4f51307ba7916eecdb8deb17f6d7b7213d19d9ed Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 11 Nov 2022 14:27:32 +0000 Subject: [PATCH 12/13] revert change to cfg_to_instructions --- Python/compile.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Python/compile.c b/Python/compile.c index 30e8c5c4b9722d..bc3543c28f1ff1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -9913,10 +9913,9 @@ cfg_to_instructions(cfg_builder *g) for (int i = 0; i < b->b_iused; i++) { struct instr *instr = &b->b_instr[i]; location loc = instr->i_loc; - int arg = instr->i_oparg; - if (HAS_TARGET(instr->i_opcode) && instr->i_target != NULL) { - arg = instr->i_target->b_label; - } + int arg = HAS_TARGET(instr->i_opcode) ? + instr->i_target->b_label : instr->i_oparg; + PyObject *inst_tuple = Py_BuildValue( "(iiiiii)", instr->i_opcode, arg, loc.lineno, loc.end_lineno, From 3657eca8e49ee0089bbd58b452ffc2be9af1ed4c Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Fri, 11 Nov 2022 15:37:15 +0000 Subject: [PATCH 13/13] static --- Python/compile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/compile.c b/Python/compile.c index bc3543c28f1ff1..88ebd0620033e2 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -628,7 +628,7 @@ compiler_setup(struct compiler *c, mod_ty mod, PyObject *filename, return 1; } -struct compiler* +static struct compiler* new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, int optimize, PyArena *arena) {