Skip to content

Commit 4db245e

Browse files
pablogsalpablogsal
authored
bpo-40334: refactor and cleanup for the PEG generators (GH-19775)
1 parent 9b64ef3 commit 4db245e

File tree

7 files changed

+292
-230
lines changed

7 files changed

+292
-230
lines changed

‎Parser/pegen/parse.c

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -648,7 +648,7 @@ file_rule(Parser *p)
648648
if (
649649
(a = statements_rule(p), 1)
650650
&&
651-
(endmarker_var = _PyPegen_endmarker_token(p))
651+
(endmarker_var = _PyPegen_expect_token(p, ENDMARKER))
652652
)
653653
{
654654
res = Module ( a , NULL , p -> arena );
@@ -712,7 +712,7 @@ eval_rule(Parser *p)
712712
&&
713713
(_loop0_1_var = _loop0_1_rule(p))
714714
&&
715-
(endmarker_var = _PyPegen_endmarker_token(p))
715+
(endmarker_var = _PyPegen_expect_token(p, ENDMARKER))
716716
)
717717
{
718718
res = Expression ( a , p -> arena );
@@ -846,7 +846,7 @@ statement_newline_rule(Parser *p)
846846
if (
847847
(a = compound_stmt_rule(p))
848848
&&
849-
(newline_var = _PyPegen_newline_token(p))
849+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
850850
)
851851
{
852852
res = _PyPegen_singleton_seq ( p , a );
@@ -872,7 +872,7 @@ statement_newline_rule(Parser *p)
872872
{ // NEWLINE
873873
void *newline_var;
874874
if (
875-
(newline_var = _PyPegen_newline_token(p))
875+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
876876
)
877877
{
878878
Token *token = _PyPegen_get_last_nonnwhitespace_token(p);
@@ -895,7 +895,7 @@ statement_newline_rule(Parser *p)
895895
{ // $
896896
void *endmarker_var;
897897
if (
898-
(endmarker_var = _PyPegen_endmarker_token(p))
898+
(endmarker_var = _PyPegen_expect_token(p, ENDMARKER))
899899
)
900900
{
901901
res = _PyPegen_interactive_exit ( p );
@@ -929,7 +929,7 @@ simple_stmt_rule(Parser *p)
929929
&&
930930
_PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 13)
931931
&&
932-
(newline_var = _PyPegen_newline_token(p))
932+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
933933
)
934934
{
935935
res = _PyPegen_singleton_seq ( p , a );
@@ -951,7 +951,7 @@ simple_stmt_rule(Parser *p)
951951
&&
952952
(opt_var = _PyPegen_expect_token(p, 13), 1)
953953
&&
954-
(newline_var = _PyPegen_newline_token(p))
954+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
955955
)
956956
{
957957
res = a;
@@ -2684,7 +2684,7 @@ for_stmt_rule(Parser *p)
26842684
void *literal;
26852685
expr_ty t;
26862686
if (
2687-
(is_async = _PyPegen_async_token(p), 1)
2687+
(is_async = _PyPegen_expect_token(p, ASYNC), 1)
26882688
&&
26892689
(keyword = _PyPegen_expect_token(p, 517))
26902690
&&
@@ -2751,7 +2751,7 @@ with_stmt_rule(Parser *p)
27512751
void *literal_1;
27522752
void *literal_2;
27532753
if (
2754-
(is_async = _PyPegen_async_token(p), 1)
2754+
(is_async = _PyPegen_expect_token(p, ASYNC), 1)
27552755
&&
27562756
(keyword = _PyPegen_expect_token(p, 519))
27572757
&&
@@ -2790,7 +2790,7 @@ with_stmt_rule(Parser *p)
27902790
void *keyword;
27912791
void *literal;
27922792
if (
2793-
(is_async = _PyPegen_async_token(p), 1)
2793+
(is_async = _PyPegen_expect_token(p, ASYNC), 1)
27942794
&&
27952795
(keyword = _PyPegen_expect_token(p, 519))
27962796
&&
@@ -3263,7 +3263,7 @@ function_def_raw_rule(Parser *p)
32633263
expr_ty n;
32643264
void *params;
32653265
if (
3266-
(is_async = _PyPegen_async_token(p), 1)
3266+
(is_async = _PyPegen_expect_token(p, ASYNC), 1)
32673267
&&
32683268
(keyword = _PyPegen_expect_token(p, 522))
32693269
&&
@@ -4002,13 +4002,13 @@ block_rule(Parser *p)
40024002
void *indent_var;
40034003
void *newline_var;
40044004
if (
4005-
(newline_var = _PyPegen_newline_token(p))
4005+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
40064006
&&
4007-
(indent_var = _PyPegen_indent_token(p))
4007+
(indent_var = _PyPegen_expect_token(p, INDENT))
40084008
&&
40094009
(a = statements_rule(p))
40104010
&&
4011-
(dedent_var = _PyPegen_dedent_token(p))
4011+
(dedent_var = _PyPegen_expect_token(p, DEDENT))
40124012
)
40134013
{
40144014
res = a;
@@ -6754,7 +6754,7 @@ await_primary_rule(Parser *p)
67546754
expr_ty a;
67556755
void *await_var;
67566756
if (
6757-
(await_var = _PyPegen_await_token(p))
6757+
(await_var = _PyPegen_expect_token(p, AWAIT))
67586758
&&
67596759
(a = primary_rule(p))
67606760
)
@@ -9919,9 +9919,9 @@ invalid_block_rule(Parser *p)
99199919
{ // NEWLINE !INDENT
99209920
void *newline_var;
99219921
if (
9922-
(newline_var = _PyPegen_newline_token(p))
9922+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
99239923
&&
9924-
_PyPegen_lookahead(0, _PyPegen_indent_token, p)
9924+
_PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, INDENT)
99259925
)
99269926
{
99279927
res = RAISE_INDENTATION_ERROR ( "expected an indented block" );
@@ -10036,7 +10036,7 @@ _loop0_1_rule(Parser *p)
1003610036
{ // NEWLINE
1003710037
void *newline_var;
1003810038
while (
10039-
(newline_var = _PyPegen_newline_token(p))
10039+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
1004010040
)
1004110041
{
1004210042
res = newline_var;
@@ -10273,7 +10273,7 @@ _tmp_6_rule(Parser *p)
1027310273
{ // ASYNC
1027410274
void *async_var;
1027510275
if (
10276-
(async_var = _PyPegen_async_token(p))
10276+
(async_var = _PyPegen_expect_token(p, ASYNC))
1027710277
)
1027810278
{
1027910279
res = async_var;
@@ -10345,7 +10345,7 @@ _tmp_8_rule(Parser *p)
1034510345
{ // ASYNC
1034610346
void *async_var;
1034710347
if (
10348-
(async_var = _PyPegen_async_token(p))
10348+
(async_var = _PyPegen_expect_token(p, ASYNC))
1034910349
)
1035010350
{
1035110351
res = async_var;
@@ -10381,7 +10381,7 @@ _tmp_9_rule(Parser *p)
1038110381
{ // ASYNC
1038210382
void *async_var;
1038310383
if (
10384-
(async_var = _PyPegen_async_token(p))
10384+
(async_var = _PyPegen_expect_token(p, ASYNC))
1038510385
)
1038610386
{
1038710387
res = async_var;
@@ -15068,7 +15068,7 @@ _tmp_128_rule(Parser *p)
1506815068
&&
1506915069
(f = named_expression_rule(p))
1507015070
&&
15071-
(newline_var = _PyPegen_newline_token(p))
15071+
(newline_var = _PyPegen_expect_token(p, NEWLINE))
1507215072
)
1507315073
{
1507415074
res = f;
@@ -15257,7 +15257,7 @@ _tmp_134_rule(Parser *p)
1525715257
void *keyword_1;
1525815258
void *y;
1525915259
if (
15260-
(y = _PyPegen_async_token(p), 1)
15260+
(y = _PyPegen_expect_token(p, ASYNC), 1)
1526115261
&&
1526215262
(keyword = _PyPegen_expect_token(p, 517))
1526315263
&&

‎Parser/pegen/pegen.c

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -692,16 +692,6 @@ _PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p)
692692
return (res != NULL) == positive;
693693
}
694694

695-
int
696-
_PyPegen_lookahead_with_string(int positive, void *(func)(Parser *, const char *), Parser *p,
697-
const char *arg)
698-
{
699-
int mark = p->mark;
700-
void *res = func(p, arg);
701-
p->mark = mark;
702-
return (res != NULL) == positive;
703-
}
704-
705695
int
706696
_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg)
707697
{
@@ -751,24 +741,6 @@ _PyPegen_get_last_nonnwhitespace_token(Parser *p)
751741
return token;
752742
}
753743

754-
void *
755-
_PyPegen_async_token(Parser *p)
756-
{
757-
return _PyPegen_expect_token(p, ASYNC);
758-
}
759-
760-
void *
761-
_PyPegen_await_token(Parser *p)
762-
{
763-
return _PyPegen_expect_token(p, AWAIT);
764-
}
765-
766-
void *
767-
_PyPegen_endmarker_token(Parser *p)
768-
{
769-
return _PyPegen_expect_token(p, ENDMARKER);
770-
}
771-
772744
expr_ty
773745
_PyPegen_name_token(Parser *p)
774746
{
@@ -794,24 +766,6 @@ _PyPegen_string_token(Parser *p)
794766
return _PyPegen_expect_token(p, STRING);
795767
}
796768

797-
void *
798-
_PyPegen_newline_token(Parser *p)
799-
{
800-
return _PyPegen_expect_token(p, NEWLINE);
801-
}
802-
803-
void *
804-
_PyPegen_indent_token(Parser *p)
805-
{
806-
return _PyPegen_expect_token(p, INDENT);
807-
}
808-
809-
void *
810-
_PyPegen_dedent_token(Parser *p)
811-
{
812-
return _PyPegen_expect_token(p, DEDENT);
813-
}
814-
815769
static PyObject *
816770
parsenumber_raw(const char *s)
817771
{

‎Parser/pegen/pegen.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,6 @@ int _PyPegen_update_memo(Parser *p, int mark, int type, void *node);
104104
int _PyPegen_is_memoized(Parser *p, int type, void *pres);
105105

106106
int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *);
107-
int _PyPegen_lookahead_with_string(int, void *(func)(Parser *, const char *), Parser *, const char *);
108107
int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int);
109108
int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *);
110109

‎Tools/peg_generator/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ dump: peg_extension/parse.c
3333
$(PYTHON) -c "from peg_extension import parse; import ast; t = parse.parse_file('$(TESTFILE)', mode=1); print(ast.dump(t))"
3434

3535
regen-metaparser: pegen/metagrammar.gram pegen/*.py
36-
$(PYTHON) -m pegen -q -c pegen/metagrammar.gram -o pegen/grammar_parser.py
36+
$(PYTHON) -m pegen -q python pegen/metagrammar.gram -o pegen/grammar_parser.py
3737

3838
# Note: These targets really depend on the generated shared object in peg_extension/parse.*.so but
3939
# this has different names in different systems so we are abusing the implicit dependency on

0 commit comments

Comments
 (0)