From 99a2b40bd5c8cbe0fe5bf818b326f2ff22aceaab Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 09:57:42 +0200 Subject: [PATCH 01/12] gh-104169: Refactor tokenizer into lexer and wrappers * The lexer, which include the actual lexeme producing logic, goes into the `lexer` directory. * The wrappers, one wrapper per input mode (file, string, utf-8, and readline), go into the `tokenizer` directory and include logic for creating a lexer instance and managing the buffer for different modes. --- Makefile.pre.in | 20 +- Parser/action_helpers.c | 1 - Parser/lexer/buffer.c | 76 ++ Parser/lexer/buffer.h | 10 + Parser/{tokenizer.c => lexer/lexer.c} | 1738 ++----------------------- Parser/lexer/lexer.h | 10 + Parser/lexer/state.c | 149 +++ Parser/{tokenizer.h => lexer/state.h} | 31 +- Parser/peg_api.c | 1 - Parser/pegen.c | 3 +- Parser/pegen_errors.c | 3 +- Parser/string_parser.c | 2 +- Parser/tokenizer/file_tokenizer.c | 470 +++++++ Parser/tokenizer/helpers.c | 552 ++++++++ Parser/tokenizer/helpers.h | 37 + Parser/tokenizer/readline_tokenizer.c | 134 ++ Parser/tokenizer/string_tokenizer.c | 129 ++ Parser/tokenizer/tokenizer.h | 14 + Parser/tokenizer/utf8_tokenizer.c | 55 + Python/Python-tokenize.c | 4 +- 20 files changed, 1777 insertions(+), 1662 deletions(-) create mode 100644 Parser/lexer/buffer.c create mode 100644 Parser/lexer/buffer.h rename Parser/{tokenizer.c => lexer/lexer.c} (52%) create mode 100644 Parser/lexer/lexer.h create mode 100644 Parser/lexer/state.c rename Parser/{tokenizer.h => lexer/state.h} (87%) create mode 100644 Parser/tokenizer/file_tokenizer.c create mode 100644 Parser/tokenizer/helpers.c create mode 100644 Parser/tokenizer/helpers.h create mode 100644 Parser/tokenizer/readline_tokenizer.c create mode 100644 Parser/tokenizer/string_tokenizer.c create mode 100644 Parser/tokenizer/tokenizer.h create mode 100644 Parser/tokenizer/utf8_tokenizer.c diff --git a/Makefile.pre.in b/Makefile.pre.in index f612f075df9abc..9755f3cfae7748 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -347,20 +347,36 @@ PEGEN_OBJS= \ Parser/string_parser.o \ Parser/peg_api.o +TOKENIZER_OBJS= \ + Parser/lexer/buffer.o \ + Parser/lexer/lexer.o \ + Parser/lexer/state.o \ + Parser/tokenizer/file_tokenizer.o \ + Parser/tokenizer/readline_tokenizer.o \ + Parser/tokenizer/string_tokenizer.o \ + Parser/tokenizer/utf8_tokenizer.o \ + Parser/tokenizer/helpers.o PEGEN_HEADERS= \ $(srcdir)/Include/internal/pycore_parser.h \ $(srcdir)/Parser/pegen.h \ $(srcdir)/Parser/string_parser.h +TOKENIZER_HEADERS= \ + Parser/lexer/buffer.h \ + Parser/lexer/lexer.h \ + Parser/lexer/state.h \ + Parser/tokenizer/tokenizer.h \ + Parser/tokenizer/helpers.h + POBJS= \ Parser/token.o \ -PARSER_OBJS= $(POBJS) $(PEGEN_OBJS) Parser/myreadline.o Parser/tokenizer.o +PARSER_OBJS= $(POBJS) $(PEGEN_OBJS) $(TOKENIZER_OBJS) Parser/myreadline.o PARSER_HEADERS= \ $(PEGEN_HEADERS) \ - $(srcdir)/Parser/tokenizer.h + $(TOKENIZER_HEADERS) ########################################################################## # Python diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index b8713a329d4ef6..fb2d56af3fc372 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1,7 +1,6 @@ #include #include "pegen.h" -#include "tokenizer.h" #include "string_parser.h" #include "pycore_runtime.h" // _PyRuntime diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c new file mode 100644 index 00000000000000..36c543b5738a51 --- /dev/null +++ b/Parser/lexer/buffer.c @@ -0,0 +1,76 @@ +#include "Python.h" +#include "errcode.h" + +#include "state.h" + +/* Traverse and remember all f-string buffers, in order to be able to restore + them after reallocating tok->buf */ +void +remember_fstring_buffers(struct tok_state *tok) +{ + int index; + tokenizer_mode *mode; + + for (index = tok->tok_mode_stack_index; index >= 0; --index) { + mode = &(tok->tok_mode_stack[index]); + mode->f_string_start_offset = mode->f_string_start - tok->buf; + mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf; + } +} + +/* Traverse and restore all f-string buffers after reallocating tok->buf */ +void +restore_fstring_buffers(struct tok_state *tok) +{ + int index; + tokenizer_mode *mode; + + for (index = tok->tok_mode_stack_index; index >= 0; --index) { + mode = &(tok->tok_mode_stack[index]); + mode->f_string_start = tok->buf + mode->f_string_start_offset; + mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset; + } +} + +/* Read a line of text from TOK into S, using the stream in TOK. + Return NULL on failure, else S. + + On entry, tok->decoding_buffer will be one of: + 1) NULL: need to call tok->decoding_readline to get a new line + 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and + stored the result in tok->decoding_buffer + 3) PyByteArrayObject *: previous call to tok_readline_recode did not have enough room + (in the s buffer) to copy entire contents of the line read + by tok->decoding_readline. tok->decoding_buffer has the overflow. + In this case, tok_readline_recode is called in a loop (with an expanded buffer) + until the buffer ends with a '\n' (or until the end of the file is + reached): see tok_nextc and its calls to tok_reserve_buf. +*/ +int +tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) +{ + Py_ssize_t cur = tok->cur - tok->buf; + Py_ssize_t oldsize = tok->inp - tok->buf; + Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); + if (newsize > tok->end - tok->buf) { + char *newbuf = tok->buf; + Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; + Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; + Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; + remember_fstring_buffers(tok); + newbuf = (char *)PyMem_Realloc(newbuf, newsize); + if (newbuf == NULL) { + tok->done = E_NOMEM; + return 0; + } + tok->buf = newbuf; + tok->cur = tok->buf + cur; + tok->inp = tok->buf + oldsize; + tok->end = tok->buf + newsize; + tok->start = start < 0 ? NULL : tok->buf + start; + tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; + tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; + restore_fstring_buffers(tok); + } + return 1; +} diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h new file mode 100644 index 00000000000000..0e4cd9d70c4d63 --- /dev/null +++ b/Parser/lexer/buffer.h @@ -0,0 +1,10 @@ +#ifndef _LEXER_BUFFER_H_ +#define _LEXER_BUFFER_H_ + +#include "pyport.h" + +void remember_fstring_buffers(struct tok_state *tok); +void restore_fstring_buffers(struct tok_state *tok); +int tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); + +#endif diff --git a/Parser/tokenizer.c b/Parser/lexer/lexer.c similarity index 52% rename from Parser/tokenizer.c rename to Parser/lexer/lexer.c index 5e3816f59af35d..dc58d9c0ce9934 100644 --- a/Parser/tokenizer.c +++ b/Parser/lexer/lexer.c @@ -1,1361 +1,58 @@ +#include "Python.h" +#include "pycore_token.h" +#include "pycore_unicodeobject.h" +#include "errcode.h" -/* Tokenizer implementation */ - -#include "Python.h" -#include "pycore_call.h" // _PyObject_CallNoArgs() - -#include "tokenizer.h" // struct tok_state -#include "errcode.h" // E_OK - -#ifdef HAVE_UNISTD_H -# include // read() -#endif - -/* Alternate tab spacing */ -#define ALTTABSIZE 1 - -#define is_potential_identifier_start(c) (\ - (c >= 'a' && c <= 'z')\ - || (c >= 'A' && c <= 'Z')\ - || c == '_'\ - || (c >= 128)) - -#define is_potential_identifier_char(c) (\ - (c >= 'a' && c <= 'z')\ - || (c >= 'A' && c <= 'Z')\ - || (c >= '0' && c <= '9')\ - || c == '_'\ - || (c >= 128)) - - -/* Don't ever change this -- it would break the portability of Python code */ -#define TABSIZE 8 - -#define MAKE_TOKEN(token_type) token_setup(tok, token, token_type, p_start, p_end) -#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ - type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) -#define ADVANCE_LINENO() \ - tok->lineno++; \ - tok->col_offset = 0; - -#define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0) -#define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0) -#ifdef Py_DEBUG -static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { - assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); - return &(tok->tok_mode_stack[tok->tok_mode_stack_index]); -} -static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { - assert(tok->tok_mode_stack_index >= 0); - assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); - return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); -} -#else -#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index])) -#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index])) -#endif - -/* Forward */ -static struct tok_state *tok_new(void); -static int tok_nextc(struct tok_state *tok); -static void tok_backup(struct tok_state *tok, int c); -static int syntaxerror(struct tok_state *tok, const char *format, ...); - -/* Spaces in this constant are treated as "zero or more spaces or tabs" when - tokenizing. */ -static const char* type_comment_prefix = "# type: "; - -/* Create and initialize a new tok_state structure */ - -static struct tok_state * -tok_new(void) -{ - struct tok_state *tok = (struct tok_state *)PyMem_Malloc( - sizeof(struct tok_state)); - if (tok == NULL) - return NULL; - tok->buf = tok->cur = tok->inp = NULL; - tok->fp_interactive = 0; - tok->interactive_src_start = NULL; - tok->interactive_src_end = NULL; - tok->start = NULL; - tok->end = NULL; - tok->done = E_OK; - tok->fp = NULL; - tok->input = NULL; - tok->tabsize = TABSIZE; - tok->indent = 0; - tok->indstack[0] = 0; - tok->atbol = 1; - tok->pendin = 0; - tok->prompt = tok->nextprompt = NULL; - tok->lineno = 0; - tok->starting_col_offset = -1; - tok->col_offset = -1; - tok->level = 0; - tok->altindstack[0] = 0; - tok->decoding_state = STATE_INIT; - tok->decoding_erred = 0; - tok->enc = NULL; - tok->encoding = NULL; - tok->cont_line = 0; - tok->filename = NULL; - tok->decoding_readline = NULL; - tok->decoding_buffer = NULL; - tok->readline = NULL; - tok->type_comments = 0; - tok->interactive_underflow = IUNDERFLOW_NORMAL; - tok->str = NULL; - tok->report_warnings = 1; - tok->tok_extra_tokens = 0; - tok->comment_newline = 0; - tok->implicit_newline = 0; - tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0}; - tok->tok_mode_stack_index = 0; -#ifdef Py_DEBUG - tok->debug = _Py_GetConfig()->parser_debug; -#endif - return tok; -} - -static char * -new_string(const char *s, Py_ssize_t len, struct tok_state *tok) -{ - char* result = (char *)PyMem_Malloc(len + 1); - if (!result) { - tok->done = E_NOMEM; - return NULL; - } - memcpy(result, s, len); - result[len] = '\0'; - return result; -} - -static char * -error_ret(struct tok_state *tok) /* XXX */ -{ - tok->decoding_erred = 1; - if ((tok->fp != NULL || tok->readline != NULL) && tok->buf != NULL) {/* see _PyTokenizer_Free */ - PyMem_Free(tok->buf); - } - tok->buf = tok->cur = tok->inp = NULL; - tok->start = NULL; - tok->end = NULL; - tok->done = E_DECODE; - return NULL; /* as if it were EOF */ -} - - -static const char * -get_normal_name(const char *s) /* for utf-8 and latin-1 */ -{ - char buf[13]; - int i; - for (i = 0; i < 12; i++) { - int c = s[i]; - if (c == '\0') - break; - else if (c == '_') - buf[i] = '-'; - else - buf[i] = Py_TOLOWER(c); - } - buf[i] = '\0'; - if (strcmp(buf, "utf-8") == 0 || - strncmp(buf, "utf-8-", 6) == 0) - return "utf-8"; - else if (strcmp(buf, "latin-1") == 0 || - strcmp(buf, "iso-8859-1") == 0 || - strcmp(buf, "iso-latin-1") == 0 || - strncmp(buf, "latin-1-", 8) == 0 || - strncmp(buf, "iso-8859-1-", 11) == 0 || - strncmp(buf, "iso-latin-1-", 12) == 0) - return "iso-8859-1"; - else - return s; -} - -/* Return the coding spec in S, or NULL if none is found. */ - -static int -get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok) -{ - Py_ssize_t i; - *spec = NULL; - /* Coding spec must be in a comment, and that comment must be - * the only statement on the source code line. */ - for (i = 0; i < size - 6; i++) { - if (s[i] == '#') - break; - if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014') - return 1; - } - for (; i < size - 6; i++) { /* XXX inefficient search */ - const char* t = s + i; - if (memcmp(t, "coding", 6) == 0) { - const char* begin = NULL; - t += 6; - if (t[0] != ':' && t[0] != '=') - continue; - do { - t++; - } while (t[0] == ' ' || t[0] == '\t'); - - begin = t; - while (Py_ISALNUM(t[0]) || - t[0] == '-' || t[0] == '_' || t[0] == '.') - t++; - - if (begin < t) { - char* r = new_string(begin, t - begin, tok); - const char* q; - if (!r) - return 0; - q = get_normal_name(r); - if (r != q) { - PyMem_Free(r); - r = new_string(q, strlen(q), tok); - if (!r) - return 0; - } - *spec = r; - break; - } - } - } - return 1; -} - -/* Check whether the line contains a coding spec. If it does, - invoke the set_readline function for the new encoding. - This function receives the tok_state and the new encoding. - Return 1 on success, 0 on failure. */ - -static int -check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, - int set_readline(struct tok_state *, const char *)) -{ - char *cs; - if (tok->cont_line) { - /* It's a continuation line, so it can't be a coding spec. */ - tok->decoding_state = STATE_NORMAL; - return 1; - } - if (!get_coding_spec(line, &cs, size, tok)) { - return 0; - } - if (!cs) { - Py_ssize_t i; - for (i = 0; i < size; i++) { - if (line[i] == '#' || line[i] == '\n' || line[i] == '\r') - break; - if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') { - /* Stop checking coding spec after a line containing - * anything except a comment. */ - tok->decoding_state = STATE_NORMAL; - break; - } - } - return 1; - } - tok->decoding_state = STATE_NORMAL; - if (tok->encoding == NULL) { - assert(tok->decoding_readline == NULL); - if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) { - error_ret(tok); - PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); - PyMem_Free(cs); - return 0; - } - tok->encoding = cs; - } else { /* then, compare cs with BOM */ - if (strcmp(tok->encoding, cs) != 0) { - error_ret(tok); - PyErr_Format(PyExc_SyntaxError, - "encoding problem: %s with BOM", cs); - PyMem_Free(cs); - return 0; - } - PyMem_Free(cs); - } - return 1; -} - -/* See whether the file starts with a BOM. If it does, - invoke the set_readline function with the new encoding. - Return 1 on success, 0 on failure. */ - -static int -check_bom(int get_char(struct tok_state *), - void unget_char(int, struct tok_state *), - int set_readline(struct tok_state *, const char *), - struct tok_state *tok) -{ - int ch1, ch2, ch3; - ch1 = get_char(tok); - tok->decoding_state = STATE_SEEK_CODING; - if (ch1 == EOF) { - return 1; - } else if (ch1 == 0xEF) { - ch2 = get_char(tok); - if (ch2 != 0xBB) { - unget_char(ch2, tok); - unget_char(ch1, tok); - return 1; - } - ch3 = get_char(tok); - if (ch3 != 0xBF) { - unget_char(ch3, tok); - unget_char(ch2, tok); - unget_char(ch1, tok); - return 1; - } - } else { - unget_char(ch1, tok); - return 1; - } - if (tok->encoding != NULL) - PyMem_Free(tok->encoding); - tok->encoding = new_string("utf-8", 5, tok); - if (!tok->encoding) - return 0; - /* No need to set_readline: input is already utf-8 */ - return 1; -} - -static int -tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) { - assert(tok->fp_interactive); - - if (!line) { - return 0; - } - - Py_ssize_t current_size = tok->interactive_src_end - tok->interactive_src_start; - Py_ssize_t line_size = strlen(line); - char last_char = line[line_size > 0 ? line_size - 1 : line_size]; - if (last_char != '\n') { - line_size += 1; - } - char* new_str = tok->interactive_src_start; - - new_str = PyMem_Realloc(new_str, current_size + line_size + 1); - if (!new_str) { - if (tok->interactive_src_start) { - PyMem_Free(tok->interactive_src_start); - } - tok->interactive_src_start = NULL; - tok->interactive_src_end = NULL; - tok->done = E_NOMEM; - return -1; - } - strcpy(new_str + current_size, line); - tok->implicit_newline = 0; - if (last_char != '\n') { - /* Last line does not end in \n, fake one */ - new_str[current_size + line_size - 1] = '\n'; - new_str[current_size + line_size] = '\0'; - tok->implicit_newline = 1; - } - tok->interactive_src_start = new_str; - tok->interactive_src_end = new_str + current_size + line_size; - return 0; -} - -/* Traverse and remember all f-string buffers, in order to be able to restore - them after reallocating tok->buf */ -static void -remember_fstring_buffers(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - mode->f_string_start_offset = mode->f_string_start - tok->buf; - mode->f_string_multi_line_start_offset = mode->f_string_multi_line_start - tok->buf; - } -} - -/* Traverse and restore all f-string buffers after reallocating tok->buf */ -static void -restore_fstring_buffers(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - mode->f_string_start = tok->buf + mode->f_string_start_offset; - mode->f_string_multi_line_start = tok->buf + mode->f_string_multi_line_start_offset; - } -} - -static int -set_fstring_expr(struct tok_state* tok, struct token *token, char c) { - assert(token != NULL); - assert(c == '}' || c == ':' || c == '!'); - tokenizer_mode *tok_mode = TOK_GET_MODE(tok); - - if (!tok_mode->f_string_debug || token->metadata) { - return 0; - } - - PyObject *res = PyUnicode_DecodeUTF8( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size - tok_mode->last_expr_end, - NULL - ); - if (!res) { - return -1; - } - token->metadata = res; - return 0; -} - -static int -update_fstring_expr(struct tok_state *tok, char cur) -{ - assert(tok->cur != NULL); - - Py_ssize_t size = strlen(tok->cur); - tokenizer_mode *tok_mode = TOK_GET_MODE(tok); - - switch (cur) { - case 0: - if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { - return 1; - } - char *new_buffer = PyMem_Realloc( - tok_mode->last_expr_buffer, - tok_mode->last_expr_size + size - ); - if (new_buffer == NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - goto error; - } - tok_mode->last_expr_buffer = new_buffer; - strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); - tok_mode->last_expr_size += size; - break; - case '{': - if (tok_mode->last_expr_buffer != NULL) { - PyMem_Free(tok_mode->last_expr_buffer); - } - tok_mode->last_expr_buffer = PyMem_Malloc(size); - if (tok_mode->last_expr_buffer == NULL) { - goto error; - } - tok_mode->last_expr_size = size; - tok_mode->last_expr_end = -1; - strncpy(tok_mode->last_expr_buffer, tok->cur, size); - break; - case '}': - case '!': - case ':': - if (tok_mode->last_expr_end == -1) { - tok_mode->last_expr_end = strlen(tok->start); - } - break; - default: - Py_UNREACHABLE(); - } - return 1; -error: - tok->done = E_NOMEM; - return 0; -} - -static void -free_fstring_expressions(struct tok_state *tok) -{ - int index; - tokenizer_mode *mode; - - for (index = tok->tok_mode_stack_index; index >= 0; --index) { - mode = &(tok->tok_mode_stack[index]); - if (mode->last_expr_buffer != NULL) { - PyMem_Free(mode->last_expr_buffer); - mode->last_expr_buffer = NULL; - mode->last_expr_size = 0; - mode->last_expr_end = -1; - } - } -} - -/* Read a line of text from TOK into S, using the stream in TOK. - Return NULL on failure, else S. - - On entry, tok->decoding_buffer will be one of: - 1) NULL: need to call tok->decoding_readline to get a new line - 2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and - stored the result in tok->decoding_buffer - 3) PyByteArrayObject *: previous call to tok_readline_recode did not have enough room - (in the s buffer) to copy entire contents of the line read - by tok->decoding_readline. tok->decoding_buffer has the overflow. - In this case, tok_readline_recode is called in a loop (with an expanded buffer) - until the buffer ends with a '\n' (or until the end of the file is - reached): see tok_nextc and its calls to tok_reserve_buf. -*/ - -static int -tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) -{ - Py_ssize_t cur = tok->cur - tok->buf; - Py_ssize_t oldsize = tok->inp - tok->buf; - Py_ssize_t newsize = oldsize + Py_MAX(size, oldsize >> 1); - if (newsize > tok->end - tok->buf) { - char *newbuf = tok->buf; - Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; - Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; - Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; - remember_fstring_buffers(tok); - newbuf = (char *)PyMem_Realloc(newbuf, newsize); - if (newbuf == NULL) { - tok->done = E_NOMEM; - return 0; - } - tok->buf = newbuf; - tok->cur = tok->buf + cur; - tok->inp = tok->buf + oldsize; - tok->end = tok->buf + newsize; - tok->start = start < 0 ? NULL : tok->buf + start; - tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; - tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; - restore_fstring_buffers(tok); - } - return 1; -} - -static inline int -contains_null_bytes(const char* str, size_t size) { - return memchr(str, 0, size) != NULL; -} - -static int -tok_readline_recode(struct tok_state *tok) { - PyObject *line; - const char *buf; - Py_ssize_t buflen; - line = tok->decoding_buffer; - if (line == NULL) { - line = PyObject_CallNoArgs(tok->decoding_readline); - if (line == NULL) { - error_ret(tok); - goto error; - } - } - else { - tok->decoding_buffer = NULL; - } - buf = PyUnicode_AsUTF8AndSize(line, &buflen); - if (buf == NULL) { - error_ret(tok); - goto error; - } - // Make room for the null terminator *and* potentially - // an extra newline character that we may need to artificially - // add. - size_t buffer_size = buflen + 2; - if (!tok_reserve_buf(tok, buffer_size)) { - goto error; - } - memcpy(tok->inp, buf, buflen); - tok->inp += buflen; - *tok->inp = '\0'; - if (tok->fp_interactive && - tok_concatenate_interactive_new_line(tok, buf) == -1) { - goto error; - } - Py_DECREF(line); - return 1; -error: - Py_XDECREF(line); - return 0; -} - -/* Set the readline function for TOK to a StreamReader's - readline function. The StreamReader is named ENC. - - This function is called from check_bom and check_coding_spec. - - ENC is usually identical to the future value of tok->encoding, - except for the (currently unsupported) case of UTF-16. - - Return 1 on success, 0 on failure. */ - -static int -fp_setreadl(struct tok_state *tok, const char* enc) -{ - PyObject *readline, *open, *stream; - int fd; - long pos; - - fd = fileno(tok->fp); - /* Due to buffering the file offset for fd can be different from the file - * position of tok->fp. If tok->fp was opened in text mode on Windows, - * its file position counts CRLF as one char and can't be directly mapped - * to the file offset for fd. Instead we step back one byte and read to - * the end of line.*/ - pos = ftell(tok->fp); - if (pos == -1 || - lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) { - PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL); - return 0; - } - - open = _PyImport_GetModuleAttrString("io", "open"); - if (open == NULL) { - return 0; - } - stream = PyObject_CallFunction(open, "isisOOO", - fd, "r", -1, enc, Py_None, Py_None, Py_False); - Py_DECREF(open); - if (stream == NULL) { - return 0; - } - - readline = PyObject_GetAttr(stream, &_Py_ID(readline)); - Py_DECREF(stream); - if (readline == NULL) { - return 0; - } - Py_XSETREF(tok->decoding_readline, readline); - - if (pos > 0) { - PyObject *bufobj = _PyObject_CallNoArgs(readline); - if (bufobj == NULL) { - return 0; - } - Py_DECREF(bufobj); - } - - return 1; -} - -/* Fetch the next byte from TOK. */ - -static int fp_getc(struct tok_state *tok) { - return getc(tok->fp); -} - -/* Unfetch the last byte back into TOK. */ - -static void fp_ungetc(int c, struct tok_state *tok) { - ungetc(c, tok->fp); -} - -/* Check whether the characters at s start a valid - UTF-8 sequence. Return the number of characters forming - the sequence if yes, 0 if not. The special cases match - those in stringlib/codecs.h:utf8_decode. -*/ -static int -valid_utf8(const unsigned char* s) -{ - int expected = 0; - int length; - if (*s < 0x80) { - /* single-byte code */ - return 1; - } - else if (*s < 0xE0) { - /* \xC2\x80-\xDF\xBF -- 0080-07FF */ - if (*s < 0xC2) { - /* invalid sequence - \x80-\xBF -- continuation byte - \xC0-\xC1 -- fake 0000-007F */ - return 0; - } - expected = 1; - } - else if (*s < 0xF0) { - /* \xE0\xA0\x80-\xEF\xBF\xBF -- 0800-FFFF */ - if (*s == 0xE0 && *(s + 1) < 0xA0) { - /* invalid sequence - \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */ - return 0; - } - else if (*s == 0xED && *(s + 1) >= 0xA0) { - /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF - will result in surrogates in range D800-DFFF. Surrogates are - not valid UTF-8 so they are rejected. - See https://www.unicode.org/versions/Unicode5.2.0/ch03.pdf - (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ - return 0; - } - expected = 2; - } - else if (*s < 0xF5) { - /* \xF0\x90\x80\x80-\xF4\x8F\xBF\xBF -- 10000-10FFFF */ - if (*(s + 1) < 0x90 ? *s == 0xF0 : *s == 0xF4) { - /* invalid sequence -- one of: - \xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF - \xF4\x90\x80\x80- -- 110000- overflow */ - return 0; - } - expected = 3; - } - else { - /* invalid start byte */ - return 0; - } - length = expected + 1; - for (; expected; expected--) - if (s[expected] < 0x80 || s[expected] >= 0xC0) - return 0; - return length; -} - -static int -ensure_utf8(char *line, struct tok_state *tok) -{ - int badchar = 0; - unsigned char *c; - int length; - for (c = (unsigned char *)line; *c; c += length) { - if (!(length = valid_utf8(c))) { - badchar = *c; - break; - } - } - if (badchar) { - PyErr_Format(PyExc_SyntaxError, - "Non-UTF-8 code starting with '\\x%.2x' " - "in file %U on line %i, " - "but no encoding declared; " - "see https://peps.python.org/pep-0263/ for details", - badchar, tok->filename, tok->lineno); - return 0; - } - return 1; -} - -/* Fetch a byte from TOK, using the string buffer. */ - -static int -buf_getc(struct tok_state *tok) { - return Py_CHARMASK(*tok->str++); -} - -/* Unfetch a byte from TOK, using the string buffer. */ - -static void -buf_ungetc(int c, struct tok_state *tok) { - tok->str--; - assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */ -} - -/* Set the readline function for TOK to ENC. For the string-based - tokenizer, this means to just record the encoding. */ - -static int -buf_setreadl(struct tok_state *tok, const char* enc) { - tok->enc = enc; - return 1; -} - -/* Return a UTF-8 encoding Python string object from the - C byte string STR, which is encoded with ENC. */ - -static PyObject * -translate_into_utf8(const char* str, const char* enc) { - PyObject *utf8; - PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL); - if (buf == NULL) - return NULL; - utf8 = PyUnicode_AsUTF8String(buf); - Py_DECREF(buf); - return utf8; -} - - -static char * -translate_newlines(const char *s, int exec_input, int preserve_crlf, - struct tok_state *tok) { - int skip_next_lf = 0; - size_t needed_length = strlen(s) + 2, final_length; - char *buf, *current; - char c = '\0'; - buf = PyMem_Malloc(needed_length); - if (buf == NULL) { - tok->done = E_NOMEM; - return NULL; - } - for (current = buf; *s; s++, current++) { - c = *s; - if (skip_next_lf) { - skip_next_lf = 0; - if (c == '\n') { - c = *++s; - if (!c) - break; - } - } - if (!preserve_crlf && c == '\r') { - skip_next_lf = 1; - c = '\n'; - } - *current = c; - } - /* If this is exec input, add a newline to the end of the string if - there isn't one already. */ - if (exec_input && c != '\n' && c != '\0') { - *current = '\n'; - current++; - } - *current = '\0'; - final_length = current - buf + 1; - if (final_length < needed_length && final_length) { - /* should never fail */ - char* result = PyMem_Realloc(buf, final_length); - if (result == NULL) { - PyMem_Free(buf); - } - buf = result; - } - return buf; -} - -/* Decode a byte string STR for use as the buffer of TOK. - Look for encoding declarations inside STR, and record them - inside TOK. */ - -static char * -decode_str(const char *input, int single, struct tok_state *tok, int preserve_crlf) -{ - PyObject* utf8 = NULL; - char *str; - const char *s; - const char *newl[2] = {NULL, NULL}; - int lineno = 0; - tok->input = str = translate_newlines(input, single, preserve_crlf, tok); - if (str == NULL) - return NULL; - tok->enc = NULL; - tok->str = str; - if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) - return error_ret(tok); - str = tok->str; /* string after BOM if any */ - assert(str); - if (tok->enc != NULL) { - utf8 = translate_into_utf8(str, tok->enc); - if (utf8 == NULL) - return error_ret(tok); - str = PyBytes_AsString(utf8); - } - for (s = str;; s++) { - if (*s == '\0') break; - else if (*s == '\n') { - assert(lineno < 2); - newl[lineno] = s; - lineno++; - if (lineno == 2) break; - } - } - tok->enc = NULL; - /* need to check line 1 and 2 separately since check_coding_spec - assumes a single line as input */ - if (newl[0]) { - if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { - return NULL; - } - if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) { - if (!check_coding_spec(newl[0]+1, newl[1] - newl[0], - tok, buf_setreadl)) - return NULL; - } - } - if (tok->enc != NULL) { - assert(utf8 == NULL); - utf8 = translate_into_utf8(str, tok->enc); - if (utf8 == NULL) - return error_ret(tok); - str = PyBytes_AS_STRING(utf8); - } - assert(tok->decoding_buffer == NULL); - tok->decoding_buffer = utf8; /* CAUTION */ - return str; -} - -/* Set up tokenizer for string */ - -struct tok_state * -_PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf) -{ - struct tok_state *tok = tok_new(); - char *decoded; - - if (tok == NULL) - return NULL; - decoded = decode_str(str, exec_input, tok, preserve_crlf); - if (decoded == NULL) { - _PyTokenizer_Free(tok); - return NULL; - } - - tok->buf = tok->cur = tok->inp = decoded; - tok->end = decoded; - return tok; -} - -struct tok_state * -_PyTokenizer_FromReadline(PyObject* readline, const char* enc, - int exec_input, int preserve_crlf) -{ - struct tok_state *tok = tok_new(); - if (tok == NULL) - return NULL; - if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { - _PyTokenizer_Free(tok); - return NULL; - } - tok->cur = tok->inp = tok->buf; - tok->end = tok->buf + BUFSIZ; - tok->fp = NULL; - if (enc != NULL) { - tok->encoding = new_string(enc, strlen(enc), tok); - if (!tok->encoding) { - _PyTokenizer_Free(tok); - return NULL; - } - } - tok->decoding_state = STATE_NORMAL; - Py_INCREF(readline); - tok->readline = readline; - return tok; -} - -/* Set up tokenizer for UTF-8 string */ - -struct tok_state * -_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) -{ - struct tok_state *tok = tok_new(); - char *translated; - if (tok == NULL) - return NULL; - tok->input = translated = translate_newlines(str, exec_input, preserve_crlf, tok); - if (translated == NULL) { - _PyTokenizer_Free(tok); - return NULL; - } - tok->decoding_state = STATE_NORMAL; - tok->enc = NULL; - tok->str = translated; - tok->encoding = new_string("utf-8", 5, tok); - if (!tok->encoding) { - _PyTokenizer_Free(tok); - return NULL; - } - - tok->buf = tok->cur = tok->inp = translated; - tok->end = translated; - return tok; -} - -/* Set up tokenizer for file */ - -struct tok_state * -_PyTokenizer_FromFile(FILE *fp, const char* enc, - const char *ps1, const char *ps2) -{ - struct tok_state *tok = tok_new(); - if (tok == NULL) - return NULL; - if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { - _PyTokenizer_Free(tok); - return NULL; - } - tok->cur = tok->inp = tok->buf; - tok->end = tok->buf + BUFSIZ; - tok->fp = fp; - tok->prompt = ps1; - tok->nextprompt = ps2; - if (enc != NULL) { - /* Must copy encoding declaration since it - gets copied into the parse tree. */ - tok->encoding = new_string(enc, strlen(enc), tok); - if (!tok->encoding) { - _PyTokenizer_Free(tok); - return NULL; - } - tok->decoding_state = STATE_NORMAL; - } - return tok; -} - -/* Free a tok_state structure */ - -void -_PyTokenizer_Free(struct tok_state *tok) -{ - if (tok->encoding != NULL) { - PyMem_Free(tok->encoding); - } - Py_XDECREF(tok->decoding_readline); - Py_XDECREF(tok->decoding_buffer); - Py_XDECREF(tok->readline); - Py_XDECREF(tok->filename); - if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { - PyMem_Free(tok->buf); - } - if (tok->input) { - PyMem_Free(tok->input); - } - if (tok->interactive_src_start != NULL) { - PyMem_Free(tok->interactive_src_start); - } - free_fstring_expressions(tok); - PyMem_Free(tok); -} - -void -_PyToken_Free(struct token *token) { - Py_XDECREF(token->metadata); -} - -void -_PyToken_Init(struct token *token) { - token->metadata = NULL; -} - -static int -tok_readline_raw(struct tok_state *tok) -{ - do { - if (!tok_reserve_buf(tok, BUFSIZ)) { - return 0; - } - int n_chars = (int)(tok->end - tok->inp); - size_t line_size = 0; - char *line = _Py_UniversalNewlineFgetsWithSize(tok->inp, n_chars, tok->fp, NULL, &line_size); - if (line == NULL) { - return 1; - } - if (tok->fp_interactive && - tok_concatenate_interactive_new_line(tok, line) == -1) { - return 0; - } - tok->inp += line_size; - if (tok->inp == tok->buf) { - return 0; - } - } while (tok->inp[-1] != '\n'); - return 1; -} - -static int -tok_readline_string(struct tok_state* tok) { - PyObject* line = NULL; - PyObject* raw_line = PyObject_CallNoArgs(tok->readline); - if (raw_line == NULL) { - if (PyErr_ExceptionMatches(PyExc_StopIteration)) { - PyErr_Clear(); - return 1; - } - error_ret(tok); - goto error; - } - if(tok->encoding != NULL) { - if (!PyBytes_Check(raw_line)) { - PyErr_Format(PyExc_TypeError, "readline() returned a non-bytes object"); - error_ret(tok); - goto error; - } - line = PyUnicode_Decode(PyBytes_AS_STRING(raw_line), PyBytes_GET_SIZE(raw_line), - tok->encoding, "replace"); - Py_CLEAR(raw_line); - if (line == NULL) { - error_ret(tok); - goto error; - } - } else { - if(!PyUnicode_Check(raw_line)) { - PyErr_Format(PyExc_TypeError, "readline() returned a non-string object"); - error_ret(tok); - goto error; - } - line = raw_line; - raw_line = NULL; - } - Py_ssize_t buflen; - const char* buf = PyUnicode_AsUTF8AndSize(line, &buflen); - if (buf == NULL) { - error_ret(tok); - goto error; - } - - // Make room for the null terminator *and* potentially - // an extra newline character that we may need to artificially - // add. - size_t buffer_size = buflen + 2; - if (!tok_reserve_buf(tok, buffer_size)) { - goto error; - } - memcpy(tok->inp, buf, buflen); - tok->inp += buflen; - *tok->inp = '\0'; - - tok->line_start = tok->cur; - Py_DECREF(line); - return 1; -error: - Py_XDECREF(raw_line); - Py_XDECREF(line); - return 0; -} - -static int -tok_underflow_string(struct tok_state *tok) { - char *end = strchr(tok->inp, '\n'); - if (end != NULL) { - end++; - } - else { - end = strchr(tok->inp, '\0'); - if (end == tok->inp) { - tok->done = E_EOF; - return 0; - } - } - if (tok->start == NULL) { - tok->buf = tok->cur; - } - tok->line_start = tok->cur; - ADVANCE_LINENO(); - tok->inp = end; - return 1; -} - -static int -tok_underflow_interactive(struct tok_state *tok) { - if (tok->interactive_underflow == IUNDERFLOW_STOP) { - tok->done = E_INTERACT_STOP; - return 1; - } - char *newtok = PyOS_Readline(tok->fp ? tok->fp : stdin, stdout, tok->prompt); - if (newtok != NULL) { - char *translated = translate_newlines(newtok, 0, 0, tok); - PyMem_Free(newtok); - if (translated == NULL) { - return 0; - } - newtok = translated; - } - if (tok->encoding && newtok && *newtok) { - /* Recode to UTF-8 */ - Py_ssize_t buflen; - const char* buf; - PyObject *u = translate_into_utf8(newtok, tok->encoding); - PyMem_Free(newtok); - if (u == NULL) { - tok->done = E_DECODE; - return 0; - } - buflen = PyBytes_GET_SIZE(u); - buf = PyBytes_AS_STRING(u); - newtok = PyMem_Malloc(buflen+1); - if (newtok == NULL) { - Py_DECREF(u); - tok->done = E_NOMEM; - return 0; - } - strcpy(newtok, buf); - Py_DECREF(u); - } - if (tok->fp_interactive && - tok_concatenate_interactive_new_line(tok, newtok) == -1) { - PyMem_Free(newtok); - return 0; - } - if (tok->nextprompt != NULL) { - tok->prompt = tok->nextprompt; - } - if (newtok == NULL) { - tok->done = E_INTR; - } - else if (*newtok == '\0') { - PyMem_Free(newtok); - tok->done = E_EOF; - } - else if (tok->start != NULL) { - Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf; - remember_fstring_buffers(tok); - size_t size = strlen(newtok); - ADVANCE_LINENO(); - if (!tok_reserve_buf(tok, size + 1)) { - PyMem_Free(tok->buf); - tok->buf = NULL; - PyMem_Free(newtok); - return 0; - } - memcpy(tok->cur, newtok, size + 1); - PyMem_Free(newtok); - tok->inp += size; - tok->multi_line_start = tok->buf + cur_multi_line_start; - restore_fstring_buffers(tok); - } - else { - remember_fstring_buffers(tok); - ADVANCE_LINENO(); - PyMem_Free(tok->buf); - tok->buf = newtok; - tok->cur = tok->buf; - tok->line_start = tok->buf; - tok->inp = strchr(tok->buf, '\0'); - tok->end = tok->inp + 1; - restore_fstring_buffers(tok); - } - if (tok->done != E_OK) { - if (tok->prompt != NULL) { - PySys_WriteStderr("\n"); - } - return 0; - } - - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { - return 0; - } - return 1; -} +#include "state.h" +#include "../tokenizer/helpers.h" -static int -tok_underflow_file(struct tok_state *tok) { - if (tok->start == NULL && !INSIDE_FSTRING(tok)) { - tok->cur = tok->inp = tok->buf; - } - if (tok->decoding_state == STATE_INIT) { - /* We have not yet determined the encoding. - If an encoding is found, use the file-pointer - reader functions from now on. */ - if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) { - error_ret(tok); - return 0; - } - assert(tok->decoding_state != STATE_INIT); - } - /* Read until '\n' or EOF */ - if (tok->decoding_readline != NULL) { - /* We already have a codec associated with this input. */ - if (!tok_readline_recode(tok)) { - return 0; - } - } - else { - /* We want a 'raw' read. */ - if (!tok_readline_raw(tok)) { - return 0; - } - } - if (tok->inp == tok->cur) { - tok->done = E_EOF; - return 0; - } - tok->implicit_newline = 0; - if (tok->inp[-1] != '\n') { - assert(tok->inp + 1 < tok->end); - /* Last line does not end in \n, fake one */ - *tok->inp++ = '\n'; - *tok->inp = '\0'; - tok->implicit_newline = 1; - } +/* Alternate tab spacing */ +#define ALTTABSIZE 1 - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { - return 0; - } +#define is_potential_identifier_start(c) (\ + (c >= 'a' && c <= 'z')\ + || (c >= 'A' && c <= 'Z')\ + || c == '_'\ + || (c >= 128)) - ADVANCE_LINENO(); - if (tok->decoding_state != STATE_NORMAL) { - if (tok->lineno > 2) { - tok->decoding_state = STATE_NORMAL; - } - else if (!check_coding_spec(tok->cur, strlen(tok->cur), - tok, fp_setreadl)) - { - return 0; - } - } - /* The default encoding is UTF-8, so make sure we don't have any - non-UTF-8 sequences in it. */ - if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { - error_ret(tok); - return 0; - } - assert(tok->done == E_OK); - return tok->done == E_OK; -} +#define is_potential_identifier_char(c) (\ + (c >= 'a' && c <= 'z')\ + || (c >= 'A' && c <= 'Z')\ + || (c >= '0' && c <= '9')\ + || c == '_'\ + || (c >= 128)) -static int -tok_underflow_readline(struct tok_state* tok) { - assert(tok->decoding_state == STATE_NORMAL); - assert(tok->fp == NULL && tok->input == NULL && tok->decoding_readline == NULL); - if (tok->start == NULL && !INSIDE_FSTRING(tok)) { - tok->cur = tok->inp = tok->buf; - } - if (!tok_readline_string(tok)) { - return 0; - } - if (tok->inp == tok->cur) { - tok->done = E_EOF; - return 0; - } - tok->implicit_newline = 0; - if (tok->inp[-1] != '\n') { - assert(tok->inp + 1 < tok->end); - /* Last line does not end in \n, fake one */ - *tok->inp++ = '\n'; - *tok->inp = '\0'; - tok->implicit_newline = 1; - } +#ifdef Py_DEBUG +static inline tokenizer_mode* TOK_GET_MODE(struct tok_state* tok) { + assert(tok->tok_mode_stack_index >= 0); + assert(tok->tok_mode_stack_index < MAXFSTRINGLEVEL); + return &(tok->tok_mode_stack[tok->tok_mode_stack_index]); +} +static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { + assert(tok->tok_mode_stack_index >= 0); + assert(tok->tok_mode_stack_index + 1 < MAXFSTRINGLEVEL); + return &(tok->tok_mode_stack[++tok->tok_mode_stack_index]); +} +#else +#define TOK_GET_MODE(tok) (&(tok->tok_mode_stack[tok->tok_mode_stack_index])) +#define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index])) +#endif - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { - return 0; - } +#define MAKE_TOKEN(token_type) token_setup(tok, token, token_type, p_start, p_end) +#define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ + type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) - ADVANCE_LINENO(); - /* The default encoding is UTF-8, so make sure we don't have any - non-UTF-8 sequences in it. */ - if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { - error_ret(tok); - return 0; - } - assert(tok->done == E_OK); - return tok->done == E_OK; -} +/* Spaces in this constant are treated as "zero or more spaces or tabs" when + tokenizing. */ +static const char* type_comment_prefix = "# type: "; -#if defined(Py_DEBUG) -static void -print_escape(FILE *f, const char *s, Py_ssize_t size) +static inline int +contains_null_bytes(const char* str, size_t size) { - if (s == NULL) { - fputs("NULL", f); - return; - } - putc('"', f); - while (size-- > 0) { - unsigned char c = *s++; - switch (c) { - case '\n': fputs("\\n", f); break; - case '\r': fputs("\\r", f); break; - case '\t': fputs("\\t", f); break; - case '\f': fputs("\\f", f); break; - case '\'': fputs("\\'", f); break; - case '"': fputs("\\\"", f); break; - default: - if (0x20 <= c && c <= 0x7f) - putc(c, f); - else - fprintf(f, "\\x%02x", c); - } - } - putc('"', f); + return memchr(str, 0, size) != NULL; } -#endif /* Get next char, updating state; error code goes into tok->done */ - static int tok_nextc(struct tok_state *tok) { @@ -1368,18 +65,7 @@ tok_nextc(struct tok_state *tok) if (tok->done != E_OK) { return EOF; } - if (tok->readline) { - rc = tok_underflow_readline(tok); - } - else if (tok->fp == NULL) { - rc = tok_underflow_string(tok); - } - else if (tok->prompt != NULL) { - rc = tok_underflow_interactive(tok); - } - else { - rc = tok_underflow_file(tok); - } + rc = tok->underflow(tok); #if defined(Py_DEBUG) if (tok->debug) { fprintf(stderr, "line[%d] = ", tok->lineno); @@ -1403,7 +89,6 @@ tok_nextc(struct tok_state *tok) } /* Back-up one character */ - static void tok_backup(struct tok_state *tok, int c) { @@ -1419,155 +104,77 @@ tok_backup(struct tok_state *tok, int c) } static int -_syntaxerror_range(struct tok_state *tok, const char *format, - int col_offset, int end_col_offset, - va_list vargs) -{ - // In release builds, we don't want to overwrite a previous error, but in debug builds we - // want to fail if we are not doing it so we can fix it. - assert(tok->done != E_ERROR); - if (tok->done == E_ERROR) { - return ERRORTOKEN; - } - PyObject *errmsg, *errtext, *args; - errmsg = PyUnicode_FromFormatV(format, vargs); - if (!errmsg) { - goto error; - } - - errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start, - "replace"); - if (!errtext) { - goto error; - } - - if (col_offset == -1) { - col_offset = (int)PyUnicode_GET_LENGTH(errtext); - } - if (end_col_offset == -1) { - end_col_offset = col_offset; - } - - Py_ssize_t line_len = strcspn(tok->line_start, "\n"); - if (line_len != tok->cur - tok->line_start) { - Py_DECREF(errtext); - errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len, - "replace"); - } - if (!errtext) { - goto error; - } - - args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, - col_offset, errtext, tok->lineno, end_col_offset); - if (args) { - PyErr_SetObject(PyExc_SyntaxError, args); - Py_DECREF(args); - } - -error: - Py_XDECREF(errmsg); - tok->done = E_ERROR; - return ERRORTOKEN; -} - -static int -syntaxerror(struct tok_state *tok, const char *format, ...) -{ - // This errors are cleaned on startup. Todo: Fix it. - va_list vargs; - va_start(vargs, format); - int ret = _syntaxerror_range(tok, format, -1, -1, vargs); - va_end(vargs); - return ret; -} - -static int -syntaxerror_known_range(struct tok_state *tok, - int col_offset, int end_col_offset, - const char *format, ...) -{ - va_list vargs; - va_start(vargs, format); - int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs); - va_end(vargs); - return ret; -} - -static int -indenterror(struct tok_state *tok) -{ - tok->done = E_TABSPACE; - tok->cur = tok->inp; - return ERRORTOKEN; -} +set_fstring_expr(struct tok_state* tok, struct token *token, char c) { + assert(token != NULL); + assert(c == '}' || c == ':' || c == '!'); + tokenizer_mode *tok_mode = TOK_GET_MODE(tok); -static int -parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) -{ - if (!tok->report_warnings) { + if (!tok_mode->f_string_debug || token->metadata) { return 0; } - PyObject *errmsg; - va_list vargs; - va_start(vargs, format); - errmsg = PyUnicode_FromFormatV(format, vargs); - va_end(vargs); - if (!errmsg) { - goto error; - } - - if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, - tok->lineno, NULL, NULL) < 0) { - if (PyErr_ExceptionMatches(category)) { - /* Replace the DeprecationWarning exception with a SyntaxError - to get a more accurate error report */ - PyErr_Clear(); - syntaxerror(tok, "%U", errmsg); - } - goto error; + PyObject *res = PyUnicode_DecodeUTF8( + tok_mode->last_expr_buffer, + tok_mode->last_expr_size - tok_mode->last_expr_end, + NULL + ); + if (!res) { + return -1; } - Py_DECREF(errmsg); + token->metadata = res; return 0; - -error: - Py_XDECREF(errmsg); - tok->done = E_ERROR; - return -1; } -static int -warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) +int +update_fstring_expr(struct tok_state *tok, char cur) { - if (!tok->report_warnings) { - return 0; - } - - PyObject *msg = PyUnicode_FromFormat( - "invalid escape sequence '\\%c'", - (char) first_invalid_escape_char - ); - - if (msg == NULL) { - return -1; - } - - if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, - tok->lineno, NULL, NULL) < 0) { - Py_DECREF(msg); + assert(tok->cur != NULL); - if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { - /* Replace the SyntaxWarning exception with a SyntaxError - to get a more accurate error report */ - PyErr_Clear(); - return syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char); - } + Py_ssize_t size = strlen(tok->cur); + tokenizer_mode *tok_mode = TOK_GET_MODE(tok); - return -1; + switch (cur) { + case 0: + if (!tok_mode->last_expr_buffer || tok_mode->last_expr_end >= 0) { + return 1; + } + char *new_buffer = PyMem_Realloc( + tok_mode->last_expr_buffer, + tok_mode->last_expr_size + size + ); + if (new_buffer == NULL) { + PyMem_Free(tok_mode->last_expr_buffer); + goto error; + } + tok_mode->last_expr_buffer = new_buffer; + strncpy(tok_mode->last_expr_buffer + tok_mode->last_expr_size, tok->cur, size); + tok_mode->last_expr_size += size; + break; + case '{': + if (tok_mode->last_expr_buffer != NULL) { + PyMem_Free(tok_mode->last_expr_buffer); + } + tok_mode->last_expr_buffer = PyMem_Malloc(size); + if (tok_mode->last_expr_buffer == NULL) { + goto error; + } + tok_mode->last_expr_size = size; + tok_mode->last_expr_end = -1; + strncpy(tok_mode->last_expr_buffer, tok->cur, size); + break; + case '}': + case '!': + case ':': + if (tok_mode->last_expr_end == -1) { + tok_mode->last_expr_end = strlen(tok->start); + } + break; + default: + Py_UNREACHABLE(); } - - Py_DECREF(msg); + return 1; +error: + tok->done = E_NOMEM; return 0; } @@ -1730,7 +337,6 @@ tok_decimal_tail(struct tok_state *tok) return c; } - static inline int tok_continuation_line(struct tok_state *tok) { int c = tok_nextc(tok); @@ -1752,43 +358,6 @@ tok_continuation_line(struct tok_state *tok) { return c; } -static int -type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, - int end_col_offset, const char *start, const char *end) -{ - token->level = tok->level; - token->lineno = token->end_lineno = tok->lineno; - token->col_offset = col_offset; - token->end_col_offset = end_col_offset; - token->start = start; - token->end = end; - return type; -} - -static int -token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) -{ - assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); - token->level = tok->level; - if (ISSTRINGLIT(type)) { - token->lineno = tok->first_lineno; - } - else { - token->lineno = tok->lineno; - } - token->end_lineno = tok->lineno; - token->col_offset = token->end_col_offset = -1; - token->start = start; - token->end = end; - - if (start != NULL && end != NULL) { - token->col_offset = tok->starting_col_offset; - token->end_col_offset = tok->col_offset; - } - return type; -} - - static int tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct token *token) { @@ -2827,7 +1396,6 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct return MAKE_TOKEN(FSTRING_MIDDLE); } - static int tok_get(struct tok_state *tok, struct token *token) { @@ -2849,103 +1417,3 @@ _PyTokenizer_Get(struct tok_state *tok, struct token *token) } return result; } - -#if defined(__wasi__) || (defined(__EMSCRIPTEN__) && (__EMSCRIPTEN_major__ >= 3)) -// fdopen() with borrowed fd. WASI does not provide dup() and Emscripten's -// dup() emulation with open() is slow. -typedef union { - void *cookie; - int fd; -} borrowed; - -static ssize_t -borrow_read(void *cookie, char *buf, size_t size) -{ - borrowed b = {.cookie = cookie}; - return read(b.fd, (void *)buf, size); -} - -static FILE * -fdopen_borrow(int fd) { - // supports only reading. seek fails. close and write are no-ops. - cookie_io_functions_t io_cb = {borrow_read, NULL, NULL, NULL}; - borrowed b = {.fd = fd}; - return fopencookie(b.cookie, "r", io_cb); -} -#else -static FILE * -fdopen_borrow(int fd) { - fd = _Py_dup(fd); - if (fd < 0) { - return NULL; - } - return fdopen(fd, "r"); -} -#endif - -/* Get the encoding of a Python file. Check for the coding cookie and check if - the file starts with a BOM. - - _PyTokenizer_FindEncodingFilename() returns NULL when it can't find the - encoding in the first or second line of the file (in which case the encoding - should be assumed to be UTF-8). - - The char* returned is malloc'ed via PyMem_Malloc() and thus must be freed - by the caller. */ - -char * -_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) -{ - struct tok_state *tok; - FILE *fp; - char *encoding = NULL; - - fp = fdopen_borrow(fd); - if (fp == NULL) { - return NULL; - } - tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL); - if (tok == NULL) { - fclose(fp); - return NULL; - } - if (filename != NULL) { - tok->filename = Py_NewRef(filename); - } - else { - tok->filename = PyUnicode_FromString(""); - if (tok->filename == NULL) { - fclose(fp); - _PyTokenizer_Free(tok); - return encoding; - } - } - struct token token; - // We don't want to report warnings here because it could cause infinite recursion - // if fetching the encoding shows a warning. - tok->report_warnings = 0; - while (tok->lineno < 2 && tok->done == E_OK) { - _PyToken_Init(&token); - _PyTokenizer_Get(tok, &token); - _PyToken_Free(&token); - } - fclose(fp); - if (tok->encoding) { - encoding = (char *)PyMem_Malloc(strlen(tok->encoding) + 1); - if (encoding) { - strcpy(encoding, tok->encoding); - } - } - _PyTokenizer_Free(tok); - return encoding; -} - -#ifdef Py_DEBUG -void -tok_dump(int type, char *start, char *end) -{ - fprintf(stderr, "%s", _PyParser_TokenNames[type]); - if (type == NAME || type == NUMBER || type == STRING || type == OP) - fprintf(stderr, "(%.*s)", (int)(end - start), start); -} -#endif // Py_DEBUG diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h new file mode 100644 index 00000000000000..154051ce0c3bc1 --- /dev/null +++ b/Parser/lexer/lexer.h @@ -0,0 +1,10 @@ +#ifndef _PY_LEXER_LEXER_H_ +#define _PY_LEXER_LEXER_H_ + +#include "state.h" + +int update_fstring_expr(struct tok_state *tok, char cur); + +extern int _PyTokenizer_Get(struct tok_state *, struct token *); + +#endif diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c new file mode 100644 index 00000000000000..809f47f21d1ad6 --- /dev/null +++ b/Parser/lexer/state.c @@ -0,0 +1,149 @@ +#include "Python.h" +#include "pycore_pystate.h" +#include "pycore_token.h" +#include "errcode.h" + +#include "state.h" + +/* Never change this */ +#define TABSIZE 8 + +/* Create and initialize a new tok_state structure */ +struct tok_state * +tok_new(void) +{ + struct tok_state *tok = (struct tok_state *)PyMem_Malloc( + sizeof(struct tok_state)); + if (tok == NULL) + return NULL; + tok->buf = tok->cur = tok->inp = NULL; + tok->fp_interactive = 0; + tok->interactive_src_start = NULL; + tok->interactive_src_end = NULL; + tok->start = NULL; + tok->end = NULL; + tok->done = E_OK; + tok->fp = NULL; + tok->input = NULL; + tok->tabsize = TABSIZE; + tok->indent = 0; + tok->indstack[0] = 0; + tok->atbol = 1; + tok->pendin = 0; + tok->prompt = tok->nextprompt = NULL; + tok->lineno = 0; + tok->starting_col_offset = -1; + tok->col_offset = -1; + tok->level = 0; + tok->altindstack[0] = 0; + tok->decoding_state = STATE_INIT; + tok->decoding_erred = 0; + tok->enc = NULL; + tok->encoding = NULL; + tok->cont_line = 0; + tok->filename = NULL; + tok->decoding_readline = NULL; + tok->decoding_buffer = NULL; + tok->readline = NULL; + tok->type_comments = 0; + tok->interactive_underflow = IUNDERFLOW_NORMAL; + tok->underflow = NULL; + tok->str = NULL; + tok->report_warnings = 1; + tok->tok_extra_tokens = 0; + tok->comment_newline = 0; + tok->implicit_newline = 0; + tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0}; + tok->tok_mode_stack_index = 0; +#ifdef Py_DEBUG + tok->debug = _Py_GetConfig()->parser_debug; +#endif + return tok; +} + +static void +free_fstring_expressions(struct tok_state *tok) +{ + int index; + tokenizer_mode *mode; + + for (index = tok->tok_mode_stack_index; index >= 0; --index) { + mode = &(tok->tok_mode_stack[index]); + if (mode->last_expr_buffer != NULL) { + PyMem_Free(mode->last_expr_buffer); + mode->last_expr_buffer = NULL; + mode->last_expr_size = 0; + mode->last_expr_end = -1; + } + } +} + +/* Free a tok_state structure */ +void +_PyTokenizer_Free(struct tok_state *tok) +{ + if (tok->encoding != NULL) { + PyMem_Free(tok->encoding); + } + Py_XDECREF(tok->decoding_readline); + Py_XDECREF(tok->decoding_buffer); + Py_XDECREF(tok->readline); + Py_XDECREF(tok->filename); + if ((tok->readline != NULL || tok->fp != NULL ) && tok->buf != NULL) { + PyMem_Free(tok->buf); + } + if (tok->input) { + PyMem_Free(tok->input); + } + if (tok->interactive_src_start != NULL) { + PyMem_Free(tok->interactive_src_start); + } + free_fstring_expressions(tok); + PyMem_Free(tok); +} + +void +_PyToken_Free(struct token *token) { + Py_XDECREF(token->metadata); +} + +void +_PyToken_Init(struct token *token) { + token->metadata = NULL; +} + +int +type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, + int end_col_offset, const char *start, const char *end) +{ + token->level = tok->level; + token->lineno = token->end_lineno = tok->lineno; + token->col_offset = col_offset; + token->end_col_offset = end_col_offset; + token->start = start; + token->end = end; + return type; +} + +int +token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) +{ + assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); + token->level = tok->level; + if (ISSTRINGLIT(type)) { + token->lineno = tok->first_lineno; + } + else { + token->lineno = tok->lineno; + } + token->end_lineno = tok->lineno; + token->col_offset = token->end_col_offset = -1; + token->start = start; + token->end = end; + + if (start != NULL && end != NULL) { + token->col_offset = tok->starting_col_offset; + token->end_col_offset = tok->col_offset; + } + return type; +} diff --git a/Parser/tokenizer.h b/Parser/lexer/state.h similarity index 87% rename from Parser/tokenizer.h rename to Parser/lexer/state.h index 11d69fc5b2e15c..606839635f02cd 100644 --- a/Parser/tokenizer.h +++ b/Parser/lexer/state.h @@ -1,19 +1,15 @@ -#ifndef Py_TOKENIZER_H -#define Py_TOKENIZER_H -#ifdef __cplusplus -extern "C" { -#endif +#ifndef _PY_LEXER_H_ +#define _PY_LEXER_H_ #include "object.h" -/* Tokenizer interface */ - -#include "pycore_token.h" /* For token types */ - #define MAXINDENT 100 /* Max indentation level */ #define MAXLEVEL 200 /* Max parentheses level */ #define MAXFSTRINGLEVEL 150 /* Max f-string nesting level */ +#define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0) +#define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0) + enum decoding_state { STATE_INIT, STATE_SEEK_CODING, @@ -118,6 +114,8 @@ struct tok_state { /* How to proceed when asked for a new token in interactive mode */ enum interactive_underflow_t interactive_underflow; + int (*underflow)(struct tok_state *); /* Function to call when buffer is empty and we need to refill it*/ + int report_warnings; // TODO: Factor this into its own thing tokenizer_mode tok_mode_stack[MAXFSTRINGLEVEL]; @@ -130,19 +128,14 @@ struct tok_state { #endif }; -extern struct tok_state *_PyTokenizer_FromString(const char *, int, int); -extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int); -extern struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int); -extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*, - const char *, const char *); +int type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, + int end_col_offset, const char *start, const char *end); +int token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); + +extern struct tok_state *tok_new(); extern void _PyTokenizer_Free(struct tok_state *); extern void _PyToken_Free(struct token *); extern void _PyToken_Init(struct token *); -extern int _PyTokenizer_Get(struct tok_state *, struct token *); -#define tok_dump _Py_tok_dump -#ifdef __cplusplus -} #endif -#endif /* !Py_TOKENIZER_H */ diff --git a/Parser/peg_api.c b/Parser/peg_api.c index 1487ac4ff856c2..447f56651af424 100644 --- a/Parser/peg_api.c +++ b/Parser/peg_api.c @@ -1,6 +1,5 @@ #include "Python.h" -#include "tokenizer.h" #include "pegen.h" mod_ty diff --git a/Parser/pegen.c b/Parser/pegen.c index bfade3446a57f7..2e0b569c20e827 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -3,7 +3,8 @@ #include "pycore_pystate.h" // _PyThreadState_GET() #include -#include "tokenizer.h" +#include "lexer/lexer.h" +#include "tokenizer/tokenizer.h" #include "pegen.h" // Internal parser functions diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index b11b0442fe96ff..2baae5289b5667 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -2,7 +2,8 @@ #include #include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject() -#include "tokenizer.h" +#include "lexer/state.h" +#include "lexer/lexer.h" #include "pegen.h" // TOKENIZER ERRORS diff --git a/Parser/string_parser.c b/Parser/string_parser.c index 72898c38b79bde..c5f421844e9c52 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -4,7 +4,7 @@ #include "pycore_bytesobject.h" // _PyBytes_DecodeEscape() #include "pycore_unicodeobject.h" // _PyUnicode_DecodeUnicodeEscapeInternal() -#include "tokenizer.h" +#include "lexer/state.h" #include "pegen.h" #include "string_parser.h" diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c new file mode 100644 index 00000000000000..535790cdb38fbf --- /dev/null +++ b/Parser/tokenizer/file_tokenizer.c @@ -0,0 +1,470 @@ +#ifdef HAVE_UNISTD_H +# include // read() +#endif + +#include "Python.h" +#include "pycore_call.h" +#include "pycore_import.h" +#include "pycore_fileutils.h" +#include "errcode.h" + +#include "helpers.h" +#include "../lexer/state.h" +#include "../lexer/lexer.h" +#include "../lexer/buffer.h" + +static int +tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) { + assert(tok->fp_interactive); + + if (!line) { + return 0; + } + + Py_ssize_t current_size = tok->interactive_src_end - tok->interactive_src_start; + Py_ssize_t line_size = strlen(line); + char last_char = line[line_size > 0 ? line_size - 1 : line_size]; + if (last_char != '\n') { + line_size += 1; + } + char* new_str = tok->interactive_src_start; + + new_str = PyMem_Realloc(new_str, current_size + line_size + 1); + if (!new_str) { + if (tok->interactive_src_start) { + PyMem_Free(tok->interactive_src_start); + } + tok->interactive_src_start = NULL; + tok->interactive_src_end = NULL; + tok->done = E_NOMEM; + return -1; + } + strcpy(new_str + current_size, line); + tok->implicit_newline = 0; + if (last_char != '\n') { + /* Last line does not end in \n, fake one */ + new_str[current_size + line_size - 1] = '\n'; + new_str[current_size + line_size] = '\0'; + tok->implicit_newline = 1; + } + tok->interactive_src_start = new_str; + tok->interactive_src_end = new_str + current_size + line_size; + return 0; +} + +static int +tok_readline_raw(struct tok_state *tok) +{ + do { + if (!tok_reserve_buf(tok, BUFSIZ)) { + return 0; + } + int n_chars = (int)(tok->end - tok->inp); + size_t line_size = 0; + char *line = _Py_UniversalNewlineFgetsWithSize(tok->inp, n_chars, tok->fp, NULL, &line_size); + if (line == NULL) { + return 1; + } + if (tok->fp_interactive && + tok_concatenate_interactive_new_line(tok, line) == -1) { + return 0; + } + tok->inp += line_size; + if (tok->inp == tok->buf) { + return 0; + } + } while (tok->inp[-1] != '\n'); + return 1; +} + +static int +tok_readline_recode(struct tok_state *tok) { + PyObject *line; + const char *buf; + Py_ssize_t buflen; + line = tok->decoding_buffer; + if (line == NULL) { + line = PyObject_CallNoArgs(tok->decoding_readline); + if (line == NULL) { + error_ret(tok); + goto error; + } + } + else { + tok->decoding_buffer = NULL; + } + buf = PyUnicode_AsUTF8AndSize(line, &buflen); + if (buf == NULL) { + error_ret(tok); + goto error; + } + // Make room for the null terminator *and* potentially + // an extra newline character that we may need to artificially + // add. + size_t buffer_size = buflen + 2; + if (!tok_reserve_buf(tok, buffer_size)) { + goto error; + } + memcpy(tok->inp, buf, buflen); + tok->inp += buflen; + *tok->inp = '\0'; + if (tok->fp_interactive && + tok_concatenate_interactive_new_line(tok, buf) == -1) { + goto error; + } + Py_DECREF(line); + return 1; +error: + Py_XDECREF(line); + return 0; +} + +/* Fetch the next byte from TOK. */ +static int fp_getc(struct tok_state *tok) { + return getc(tok->fp); +} + +/* Unfetch the last byte back into TOK. */ +static void fp_ungetc(int c, struct tok_state *tok) { + ungetc(c, tok->fp); +} + +/* Set the readline function for TOK to a StreamReader's + readline function. The StreamReader is named ENC. + + This function is called from check_bom and check_coding_spec. + + ENC is usually identical to the future value of tok->encoding, + except for the (currently unsupported) case of UTF-16. + + Return 1 on success, 0 on failure. */ +static int +fp_setreadl(struct tok_state *tok, const char* enc) +{ + PyObject *readline, *open, *stream; + int fd; + long pos; + + fd = fileno(tok->fp); + /* Due to buffering the file offset for fd can be different from the file + * position of tok->fp. If tok->fp was opened in text mode on Windows, + * its file position counts CRLF as one char and can't be directly mapped + * to the file offset for fd. Instead we step back one byte and read to + * the end of line.*/ + pos = ftell(tok->fp); + if (pos == -1 || + lseek(fd, (off_t)(pos > 0 ? pos - 1 : pos), SEEK_SET) == (off_t)-1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, NULL); + return 0; + } + + open = _PyImport_GetModuleAttrString("io", "open"); + if (open == NULL) { + return 0; + } + stream = PyObject_CallFunction(open, "isisOOO", + fd, "r", -1, enc, Py_None, Py_None, Py_False); + Py_DECREF(open); + if (stream == NULL) { + return 0; + } + + readline = PyObject_GetAttr(stream, &_Py_ID(readline)); + Py_DECREF(stream); + if (readline == NULL) { + return 0; + } + Py_XSETREF(tok->decoding_readline, readline); + + if (pos > 0) { + PyObject *bufobj = _PyObject_CallNoArgs(readline); + if (bufobj == NULL) { + return 0; + } + Py_DECREF(bufobj); + } + + return 1; +} + +static int +tok_underflow_interactive(struct tok_state *tok) { + if (tok->interactive_underflow == IUNDERFLOW_STOP) { + tok->done = E_INTERACT_STOP; + return 1; + } + char *newtok = PyOS_Readline(tok->fp ? tok->fp : stdin, stdout, tok->prompt); + if (newtok != NULL) { + char *translated = translate_newlines(newtok, 0, 0, tok); + PyMem_Free(newtok); + if (translated == NULL) { + return 0; + } + newtok = translated; + } + if (tok->encoding && newtok && *newtok) { + /* Recode to UTF-8 */ + Py_ssize_t buflen; + const char* buf; + PyObject *u = translate_into_utf8(newtok, tok->encoding); + PyMem_Free(newtok); + if (u == NULL) { + tok->done = E_DECODE; + return 0; + } + buflen = PyBytes_GET_SIZE(u); + buf = PyBytes_AS_STRING(u); + newtok = PyMem_Malloc(buflen+1); + if (newtok == NULL) { + Py_DECREF(u); + tok->done = E_NOMEM; + return 0; + } + strcpy(newtok, buf); + Py_DECREF(u); + } + if (tok->fp_interactive && + tok_concatenate_interactive_new_line(tok, newtok) == -1) { + PyMem_Free(newtok); + return 0; + } + if (tok->nextprompt != NULL) { + tok->prompt = tok->nextprompt; + } + if (newtok == NULL) { + tok->done = E_INTR; + } + else if (*newtok == '\0') { + PyMem_Free(newtok); + tok->done = E_EOF; + } + else if (tok->start != NULL) { + Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf; + remember_fstring_buffers(tok); + size_t size = strlen(newtok); + ADVANCE_LINENO(); + if (!tok_reserve_buf(tok, size + 1)) { + PyMem_Free(tok->buf); + tok->buf = NULL; + PyMem_Free(newtok); + return 0; + } + memcpy(tok->cur, newtok, size + 1); + PyMem_Free(newtok); + tok->inp += size; + tok->multi_line_start = tok->buf + cur_multi_line_start; + restore_fstring_buffers(tok); + } + else { + remember_fstring_buffers(tok); + ADVANCE_LINENO(); + PyMem_Free(tok->buf); + tok->buf = newtok; + tok->cur = tok->buf; + tok->line_start = tok->buf; + tok->inp = strchr(tok->buf, '\0'); + tok->end = tok->inp + 1; + restore_fstring_buffers(tok); + } + if (tok->done != E_OK) { + if (tok->prompt != NULL) { + PySys_WriteStderr("\n"); + } + return 0; + } + + if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + return 0; + } + return 1; +} + +static int +tok_underflow_file(struct tok_state *tok) { + if (tok->start == NULL && !INSIDE_FSTRING(tok)) { + tok->cur = tok->inp = tok->buf; + } + if (tok->decoding_state == STATE_INIT) { + /* We have not yet determined the encoding. + If an encoding is found, use the file-pointer + reader functions from now on. */ + if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) { + error_ret(tok); + return 0; + } + assert(tok->decoding_state != STATE_INIT); + } + /* Read until '\n' or EOF */ + if (tok->decoding_readline != NULL) { + /* We already have a codec associated with this input. */ + if (!tok_readline_recode(tok)) { + return 0; + } + } + else { + /* We want a 'raw' read. */ + if (!tok_readline_raw(tok)) { + return 0; + } + } + if (tok->inp == tok->cur) { + tok->done = E_EOF; + return 0; + } + tok->implicit_newline = 0; + if (tok->inp[-1] != '\n') { + assert(tok->inp + 1 < tok->end); + /* Last line does not end in \n, fake one */ + *tok->inp++ = '\n'; + *tok->inp = '\0'; + tok->implicit_newline = 1; + } + + if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + return 0; + } + + ADVANCE_LINENO(); + if (tok->decoding_state != STATE_NORMAL) { + if (tok->lineno > 2) { + tok->decoding_state = STATE_NORMAL; + } + else if (!check_coding_spec(tok->cur, strlen(tok->cur), + tok, fp_setreadl)) + { + return 0; + } + } + /* The default encoding is UTF-8, so make sure we don't have any + non-UTF-8 sequences in it. */ + if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { + error_ret(tok); + return 0; + } + assert(tok->done == E_OK); + return tok->done == E_OK; +} + +/* Set up tokenizer for file */ +struct tok_state * +_PyTokenizer_FromFile(FILE *fp, const char* enc, + const char *ps1, const char *ps2) +{ + struct tok_state *tok = tok_new(); + if (tok == NULL) + return NULL; + if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->cur = tok->inp = tok->buf; + tok->end = tok->buf + BUFSIZ; + tok->fp = fp; + tok->prompt = ps1; + tok->nextprompt = ps2; + if (ps1 || ps2) { + tok->underflow = &tok_underflow_interactive; + } else { + tok->underflow = &tok_underflow_file; + } + if (enc != NULL) { + /* Must copy encoding declaration since it + gets copied into the parse tree. */ + tok->encoding = new_string(enc, strlen(enc), tok); + if (!tok->encoding) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->decoding_state = STATE_NORMAL; + } + return tok; +} + +#if defined(__wasi__) || (defined(__EMSCRIPTEN__) && (__EMSCRIPTEN_major__ >= 3)) +// fdopen() with borrowed fd. WASI does not provide dup() and Emscripten's +// dup() emulation with open() is slow. +typedef union { + void *cookie; + int fd; +} borrowed; + +static ssize_t +borrow_read(void *cookie, char *buf, size_t size) +{ + borrowed b = {.cookie = cookie}; + return read(b.fd, (void *)buf, size); +} + +static FILE * +fdopen_borrow(int fd) { + // supports only reading. seek fails. close and write are no-ops. + cookie_io_functions_t io_cb = {borrow_read, NULL, NULL, NULL}; + borrowed b = {.fd = fd}; + return fopencookie(b.cookie, "r", io_cb); +} +#else +static FILE * +fdopen_borrow(int fd) { + fd = _Py_dup(fd); + if (fd < 0) { + return NULL; + } + return fdopen(fd, "r"); +} +#endif + +/* Get the encoding of a Python file. Check for the coding cookie and check if + the file starts with a BOM. + + _PyTokenizer_FindEncodingFilename() returns NULL when it can't find the + encoding in the first or second line of the file (in which case the encoding + should be assumed to be UTF-8). + + The char* returned is malloc'ed via PyMem_Malloc() and thus must be freed + by the caller. */ +char * +_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename) +{ + struct tok_state *tok; + FILE *fp; + char *encoding = NULL; + + fp = fdopen_borrow(fd); + if (fp == NULL) { + return NULL; + } + tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL); + if (tok == NULL) { + fclose(fp); + return NULL; + } + if (filename != NULL) { + tok->filename = Py_NewRef(filename); + } + else { + tok->filename = PyUnicode_FromString(""); + if (tok->filename == NULL) { + fclose(fp); + _PyTokenizer_Free(tok); + return encoding; + } + } + struct token token; + // We don't want to report warnings here because it could cause infinite recursion + // if fetching the encoding shows a warning. + tok->report_warnings = 0; + while (tok->lineno < 2 && tok->done == E_OK) { + _PyToken_Init(&token); + _PyTokenizer_Get(tok, &token); + _PyToken_Free(&token); + } + fclose(fp); + if (tok->encoding) { + encoding = (char *)PyMem_Malloc(strlen(tok->encoding) + 1); + if (encoding) { + strcpy(encoding, tok->encoding); + } + } + _PyTokenizer_Free(tok); + return encoding; +} diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c new file mode 100644 index 00000000000000..4a51c44576b13b --- /dev/null +++ b/Parser/tokenizer/helpers.c @@ -0,0 +1,552 @@ +#include "Python.h" +#include "errcode.h" +#include "pycore_token.h" + +#include "../lexer/state.h" + + +/* ############## ERRORS ############## */ + +static int +_syntaxerror_range(struct tok_state *tok, const char *format, + int col_offset, int end_col_offset, + va_list vargs) +{ + // In release builds, we don't want to overwrite a previous error, but in debug builds we + // want to fail if we are not doing it so we can fix it. + assert(tok->done != E_ERROR); + if (tok->done == E_ERROR) { + return ERRORTOKEN; + } + PyObject *errmsg, *errtext, *args; + errmsg = PyUnicode_FromFormatV(format, vargs); + if (!errmsg) { + goto error; + } + + errtext = PyUnicode_DecodeUTF8(tok->line_start, tok->cur - tok->line_start, + "replace"); + if (!errtext) { + goto error; + } + + if (col_offset == -1) { + col_offset = (int)PyUnicode_GET_LENGTH(errtext); + } + if (end_col_offset == -1) { + end_col_offset = col_offset; + } + + Py_ssize_t line_len = strcspn(tok->line_start, "\n"); + if (line_len != tok->cur - tok->line_start) { + Py_DECREF(errtext); + errtext = PyUnicode_DecodeUTF8(tok->line_start, line_len, + "replace"); + } + if (!errtext) { + goto error; + } + + args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, + col_offset, errtext, tok->lineno, end_col_offset); + if (args) { + PyErr_SetObject(PyExc_SyntaxError, args); + Py_DECREF(args); + } + +error: + Py_XDECREF(errmsg); + tok->done = E_ERROR; + return ERRORTOKEN; +} + +int +syntaxerror(struct tok_state *tok, const char *format, ...) +{ + // This errors are cleaned on startup. Todo: Fix it. + va_list vargs; + va_start(vargs, format); + int ret = _syntaxerror_range(tok, format, -1, -1, vargs); + va_end(vargs); + return ret; +} + +int +syntaxerror_known_range(struct tok_state *tok, + int col_offset, int end_col_offset, + const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + int ret = _syntaxerror_range(tok, format, col_offset, end_col_offset, vargs); + va_end(vargs); + return ret; +} + +int +indenterror(struct tok_state *tok) +{ + tok->done = E_TABSPACE; + tok->cur = tok->inp; + return ERRORTOKEN; +} + +char * +error_ret(struct tok_state *tok) /* XXX */ +{ + tok->decoding_erred = 1; + if ((tok->fp != NULL || tok->readline != NULL) && tok->buf != NULL) {/* see _PyTokenizer_Free */ + PyMem_Free(tok->buf); + } + tok->buf = tok->cur = tok->inp = NULL; + tok->start = NULL; + tok->end = NULL; + tok->done = E_DECODE; + return NULL; /* as if it were EOF */ +} + +int +warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) +{ + if (!tok->report_warnings) { + return 0; + } + + PyObject *msg = PyUnicode_FromFormat( + "invalid escape sequence '\\%c'", + (char) first_invalid_escape_char + ); + + if (msg == NULL) { + return -1; + } + + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, tok->filename, + tok->lineno, NULL, NULL) < 0) { + Py_DECREF(msg); + + if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + /* Replace the SyntaxWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + return syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char); + } + + return -1; + } + + Py_DECREF(msg); + return 0; +} + +int +parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) +{ + if (!tok->report_warnings) { + return 0; + } + + PyObject *errmsg; + va_list vargs; + va_start(vargs, format); + errmsg = PyUnicode_FromFormatV(format, vargs); + va_end(vargs); + if (!errmsg) { + goto error; + } + + if (PyErr_WarnExplicitObject(category, errmsg, tok->filename, + tok->lineno, NULL, NULL) < 0) { + if (PyErr_ExceptionMatches(category)) { + /* Replace the DeprecationWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + syntaxerror(tok, "%U", errmsg); + } + goto error; + } + Py_DECREF(errmsg); + return 0; + +error: + Py_XDECREF(errmsg); + tok->done = E_ERROR; + return -1; +} + + +/* ############## STRING MANIPULATION ############## */ + +char * +new_string(const char *s, Py_ssize_t len, struct tok_state *tok) +{ + char* result = (char *)PyMem_Malloc(len + 1); + if (!result) { + tok->done = E_NOMEM; + return NULL; + } + memcpy(result, s, len); + result[len] = '\0'; + return result; +} + +PyObject * +translate_into_utf8(const char* str, const char* enc) { + PyObject *utf8; + PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL); + if (buf == NULL) + return NULL; + utf8 = PyUnicode_AsUTF8String(buf); + Py_DECREF(buf); + return utf8; +} + +char * +translate_newlines(const char *s, int exec_input, int preserve_crlf, + struct tok_state *tok) { + int skip_next_lf = 0; + size_t needed_length = strlen(s) + 2, final_length; + char *buf, *current; + char c = '\0'; + buf = PyMem_Malloc(needed_length); + if (buf == NULL) { + tok->done = E_NOMEM; + return NULL; + } + for (current = buf; *s; s++, current++) { + c = *s; + if (skip_next_lf) { + skip_next_lf = 0; + if (c == '\n') { + c = *++s; + if (!c) + break; + } + } + if (!preserve_crlf && c == '\r') { + skip_next_lf = 1; + c = '\n'; + } + *current = c; + } + /* If this is exec input, add a newline to the end of the string if + there isn't one already. */ + if (exec_input && c != '\n' && c != '\0') { + *current = '\n'; + current++; + } + *current = '\0'; + final_length = current - buf + 1; + if (final_length < needed_length && final_length) { + /* should never fail */ + char* result = PyMem_Realloc(buf, final_length); + if (result == NULL) { + PyMem_Free(buf); + } + buf = result; + } + return buf; +} + +/* ############## ENCODING STUFF ############## */ + + +/* See whether the file starts with a BOM. If it does, + invoke the set_readline function with the new encoding. + Return 1 on success, 0 on failure. */ +int +check_bom(int get_char(struct tok_state *), + void unget_char(int, struct tok_state *), + int set_readline(struct tok_state *, const char *), + struct tok_state *tok) +{ + int ch1, ch2, ch3; + ch1 = get_char(tok); + tok->decoding_state = STATE_SEEK_CODING; + if (ch1 == EOF) { + return 1; + } else if (ch1 == 0xEF) { + ch2 = get_char(tok); + if (ch2 != 0xBB) { + unget_char(ch2, tok); + unget_char(ch1, tok); + return 1; + } + ch3 = get_char(tok); + if (ch3 != 0xBF) { + unget_char(ch3, tok); + unget_char(ch2, tok); + unget_char(ch1, tok); + return 1; + } + } else { + unget_char(ch1, tok); + return 1; + } + if (tok->encoding != NULL) + PyMem_Free(tok->encoding); + tok->encoding = new_string("utf-8", 5, tok); + if (!tok->encoding) + return 0; + /* No need to set_readline: input is already utf-8 */ + return 1; +} + +static const char * +get_normal_name(const char *s) /* for utf-8 and latin-1 */ +{ + char buf[13]; + int i; + for (i = 0; i < 12; i++) { + int c = s[i]; + if (c == '\0') + break; + else if (c == '_') + buf[i] = '-'; + else + buf[i] = Py_TOLOWER(c); + } + buf[i] = '\0'; + if (strcmp(buf, "utf-8") == 0 || + strncmp(buf, "utf-8-", 6) == 0) + return "utf-8"; + else if (strcmp(buf, "latin-1") == 0 || + strcmp(buf, "iso-8859-1") == 0 || + strcmp(buf, "iso-latin-1") == 0 || + strncmp(buf, "latin-1-", 8) == 0 || + strncmp(buf, "iso-8859-1-", 11) == 0 || + strncmp(buf, "iso-latin-1-", 12) == 0) + return "iso-8859-1"; + else + return s; +} + +/* Return the coding spec in S, or NULL if none is found. */ +static int +get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *tok) +{ + Py_ssize_t i; + *spec = NULL; + /* Coding spec must be in a comment, and that comment must be + * the only statement on the source code line. */ + for (i = 0; i < size - 6; i++) { + if (s[i] == '#') + break; + if (s[i] != ' ' && s[i] != '\t' && s[i] != '\014') + return 1; + } + for (; i < size - 6; i++) { /* XXX inefficient search */ + const char* t = s + i; + if (memcmp(t, "coding", 6) == 0) { + const char* begin = NULL; + t += 6; + if (t[0] != ':' && t[0] != '=') + continue; + do { + t++; + } while (t[0] == ' ' || t[0] == '\t'); + + begin = t; + while (Py_ISALNUM(t[0]) || + t[0] == '-' || t[0] == '_' || t[0] == '.') + t++; + + if (begin < t) { + char* r = new_string(begin, t - begin, tok); + const char* q; + if (!r) + return 0; + q = get_normal_name(r); + if (r != q) { + PyMem_Free(r); + r = new_string(q, strlen(q), tok); + if (!r) + return 0; + } + *spec = r; + break; + } + } + } + return 1; +} + +/* Check whether the line contains a coding spec. If it does, + invoke the set_readline function for the new encoding. + This function receives the tok_state and the new encoding. + Return 1 on success, 0 on failure. */ +int +check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, + int set_readline(struct tok_state *, const char *)) +{ + char *cs; + if (tok->cont_line) { + /* It's a continuation line, so it can't be a coding spec. */ + tok->decoding_state = STATE_NORMAL; + return 1; + } + if (!get_coding_spec(line, &cs, size, tok)) { + return 0; + } + if (!cs) { + Py_ssize_t i; + for (i = 0; i < size; i++) { + if (line[i] == '#' || line[i] == '\n' || line[i] == '\r') + break; + if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') { + /* Stop checking coding spec after a line containing + * anything except a comment. */ + tok->decoding_state = STATE_NORMAL; + break; + } + } + return 1; + } + tok->decoding_state = STATE_NORMAL; + if (tok->encoding == NULL) { + assert(tok->decoding_readline == NULL); + if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) { + error_ret(tok); + PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); + PyMem_Free(cs); + return 0; + } + tok->encoding = cs; + } else { /* then, compare cs with BOM */ + if (strcmp(tok->encoding, cs) != 0) { + error_ret(tok); + PyErr_Format(PyExc_SyntaxError, + "encoding problem: %s with BOM", cs); + PyMem_Free(cs); + return 0; + } + PyMem_Free(cs); + } + return 1; +} + +/* Check whether the characters at s start a valid + UTF-8 sequence. Return the number of characters forming + the sequence if yes, 0 if not. The special cases match + those in stringlib/codecs.h:utf8_decode. +*/ +static int +valid_utf8(const unsigned char* s) +{ + int expected = 0; + int length; + if (*s < 0x80) { + /* single-byte code */ + return 1; + } + else if (*s < 0xE0) { + /* \xC2\x80-\xDF\xBF -- 0080-07FF */ + if (*s < 0xC2) { + /* invalid sequence + \x80-\xBF -- continuation byte + \xC0-\xC1 -- fake 0000-007F */ + return 0; + } + expected = 1; + } + else if (*s < 0xF0) { + /* \xE0\xA0\x80-\xEF\xBF\xBF -- 0800-FFFF */ + if (*s == 0xE0 && *(s + 1) < 0xA0) { + /* invalid sequence + \xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */ + return 0; + } + else if (*s == 0xED && *(s + 1) >= 0xA0) { + /* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF + will result in surrogates in range D800-DFFF. Surrogates are + not valid UTF-8 so they are rejected. + See https://www.unicode.org/versions/Unicode5.2.0/ch03.pdf + (table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */ + return 0; + } + expected = 2; + } + else if (*s < 0xF5) { + /* \xF0\x90\x80\x80-\xF4\x8F\xBF\xBF -- 10000-10FFFF */ + if (*(s + 1) < 0x90 ? *s == 0xF0 : *s == 0xF4) { + /* invalid sequence -- one of: + \xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF + \xF4\x90\x80\x80- -- 110000- overflow */ + return 0; + } + expected = 3; + } + else { + /* invalid start byte */ + return 0; + } + length = expected + 1; + for (; expected; expected--) + if (s[expected] < 0x80 || s[expected] >= 0xC0) + return 0; + return length; +} + +int +ensure_utf8(char *line, struct tok_state *tok) +{ + int badchar = 0; + unsigned char *c; + int length; + for (c = (unsigned char *)line; *c; c += length) { + if (!(length = valid_utf8(c))) { + badchar = *c; + break; + } + } + if (badchar) { + PyErr_Format(PyExc_SyntaxError, + "Non-UTF-8 code starting with '\\x%.2x' " + "in file %U on line %i, " + "but no encoding declared; " + "see https://peps.python.org/pep-0263/ for details", + badchar, tok->filename, tok->lineno); + return 0; + } + return 1; +} + + +/* ############## DEBUGGING STUFF ############## */ + +#if defined(Py_DEBUG) +void +print_escape(FILE *f, const char *s, Py_ssize_t size) +{ + if (s == NULL) { + fputs("NULL", f); + return; + } + putc('"', f); + while (size-- > 0) { + unsigned char c = *s++; + switch (c) { + case '\n': fputs("\\n", f); break; + case '\r': fputs("\\r", f); break; + case '\t': fputs("\\t", f); break; + case '\f': fputs("\\f", f); break; + case '\'': fputs("\\'", f); break; + case '"': fputs("\\\"", f); break; + default: + if (0x20 <= c && c <= 0x7f) + putc(c, f); + else + fprintf(f, "\\x%02x", c); + } + } + putc('"', f); +} + +void +tok_dump(int type, char *start, char *end) +{ + fprintf(stderr, "%s", _PyParser_TokenNames[type]); + if (type == NAME || type == NUMBER || type == STRING || type == OP) + fprintf(stderr, "(%.*s)", (int)(end - start), start); +} +#endif diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h new file mode 100644 index 00000000000000..7bb9fa15895500 --- /dev/null +++ b/Parser/tokenizer/helpers.h @@ -0,0 +1,37 @@ +#ifndef _PY_TOKENIZER_HELPERS_H_ +#define _PY_TOKENIZER_HELPERS_H_ + +#include "Python.h" + +#include "../lexer/state.h" + +#define ADVANCE_LINENO() \ + tok->lineno++; \ + tok->col_offset = 0; + +int syntaxerror(struct tok_state *tok, const char *format, ...); +int syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); +int indenterror(struct tok_state *tok); +int warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); +int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); +char *error_ret(struct tok_state *tok); + +char *new_string(const char *s, Py_ssize_t len, struct tok_state *tok); +char *translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); +PyObject *translate_into_utf8(const char* str, const char* enc); + +int check_bom(int get_char(struct tok_state *), + void unget_char(int, struct tok_state *), + int set_readline(struct tok_state *, const char *), + struct tok_state *tok); +int check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, + int set_readline(struct tok_state *, const char *)); +int ensure_utf8(char *line, struct tok_state *tok); + +#if Py_DEBUG +void print_escape(FILE *f, const char *s, Py_ssize_t size); +void tok_dump(int type, char *start, char *end); +#endif + + +#endif diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c new file mode 100644 index 00000000000000..892fa4691506cb --- /dev/null +++ b/Parser/tokenizer/readline_tokenizer.c @@ -0,0 +1,134 @@ +#include "Python.h" +#include "errcode.h" + +#include "helpers.h" +#include "../lexer/lexer.h" +#include "../lexer/state.h" +#include "../lexer/buffer.h" + +static int +tok_readline_string(struct tok_state* tok) { + PyObject* line = NULL; + PyObject* raw_line = PyObject_CallNoArgs(tok->readline); + if (raw_line == NULL) { + if (PyErr_ExceptionMatches(PyExc_StopIteration)) { + PyErr_Clear(); + return 1; + } + error_ret(tok); + goto error; + } + if(tok->encoding != NULL) { + if (!PyBytes_Check(raw_line)) { + PyErr_Format(PyExc_TypeError, "readline() returned a non-bytes object"); + error_ret(tok); + goto error; + } + line = PyUnicode_Decode(PyBytes_AS_STRING(raw_line), PyBytes_GET_SIZE(raw_line), + tok->encoding, "replace"); + Py_CLEAR(raw_line); + if (line == NULL) { + error_ret(tok); + goto error; + } + } else { + if(!PyUnicode_Check(raw_line)) { + PyErr_Format(PyExc_TypeError, "readline() returned a non-string object"); + error_ret(tok); + goto error; + } + line = raw_line; + raw_line = NULL; + } + Py_ssize_t buflen; + const char* buf = PyUnicode_AsUTF8AndSize(line, &buflen); + if (buf == NULL) { + error_ret(tok); + goto error; + } + + // Make room for the null terminator *and* potentially + // an extra newline character that we may need to artificially + // add. + size_t buffer_size = buflen + 2; + if (!tok_reserve_buf(tok, buffer_size)) { + goto error; + } + memcpy(tok->inp, buf, buflen); + tok->inp += buflen; + *tok->inp = '\0'; + + tok->line_start = tok->cur; + Py_DECREF(line); + return 1; +error: + Py_XDECREF(raw_line); + Py_XDECREF(line); + return 0; +} + +static int +tok_underflow_readline(struct tok_state* tok) { + assert(tok->decoding_state == STATE_NORMAL); + assert(tok->fp == NULL && tok->input == NULL && tok->decoding_readline == NULL); + if (tok->start == NULL && !INSIDE_FSTRING(tok)) { + tok->cur = tok->inp = tok->buf; + } + if (!tok_readline_string(tok)) { + return 0; + } + if (tok->inp == tok->cur) { + tok->done = E_EOF; + return 0; + } + tok->implicit_newline = 0; + if (tok->inp[-1] != '\n') { + assert(tok->inp + 1 < tok->end); + /* Last line does not end in \n, fake one */ + *tok->inp++ = '\n'; + *tok->inp = '\0'; + tok->implicit_newline = 1; + } + + if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + return 0; + } + + ADVANCE_LINENO(); + /* The default encoding is UTF-8, so make sure we don't have any + non-UTF-8 sequences in it. */ + if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { + error_ret(tok); + return 0; + } + assert(tok->done == E_OK); + return tok->done == E_OK; +} + +struct tok_state * +_PyTokenizer_FromReadline(PyObject* readline, const char* enc, + int exec_input, int preserve_crlf) +{ + struct tok_state *tok = tok_new(); + if (tok == NULL) + return NULL; + if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->cur = tok->inp = tok->buf; + tok->end = tok->buf + BUFSIZ; + tok->fp = NULL; + if (enc != NULL) { + tok->encoding = new_string(enc, strlen(enc), tok); + if (!tok->encoding) { + _PyTokenizer_Free(tok); + return NULL; + } + } + tok->decoding_state = STATE_NORMAL; + tok->underflow = &tok_underflow_readline; + Py_INCREF(readline); + tok->readline = readline; + return tok; +} \ No newline at end of file diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c new file mode 100644 index 00000000000000..7e48a568eb0c46 --- /dev/null +++ b/Parser/tokenizer/string_tokenizer.c @@ -0,0 +1,129 @@ +#include "Python.h" +#include "errcode.h" + +#include "helpers.h" +#include "../lexer/state.h" + +static int +tok_underflow_string(struct tok_state *tok) { + char *end = strchr(tok->inp, '\n'); + if (end != NULL) { + end++; + } + else { + end = strchr(tok->inp, '\0'); + if (end == tok->inp) { + tok->done = E_EOF; + return 0; + } + } + if (tok->start == NULL) { + tok->buf = tok->cur; + } + tok->line_start = tok->cur; + ADVANCE_LINENO(); + tok->inp = end; + return 1; +} + +/* Fetch a byte from TOK, using the string buffer. */ +static int +buf_getc(struct tok_state *tok) { + return Py_CHARMASK(*tok->str++); +} + +/* Unfetch a byte from TOK, using the string buffer. */ +static void +buf_ungetc(int c, struct tok_state *tok) { + tok->str--; + assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */ +} + +/* Set the readline function for TOK to ENC. For the string-based + tokenizer, this means to just record the encoding. */ +static int +buf_setreadl(struct tok_state *tok, const char* enc) { + tok->enc = enc; + return 1; +} + +/* Decode a byte string STR for use as the buffer of TOK. + Look for encoding declarations inside STR, and record them + inside TOK. */ +static char * +decode_str(const char *input, int single, struct tok_state *tok, int preserve_crlf) +{ + PyObject* utf8 = NULL; + char *str; + const char *s; + const char *newl[2] = {NULL, NULL}; + int lineno = 0; + tok->input = str = translate_newlines(input, single, preserve_crlf, tok); + if (str == NULL) + return NULL; + tok->enc = NULL; + tok->str = str; + if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) + return error_ret(tok); + str = tok->str; /* string after BOM if any */ + assert(str); + if (tok->enc != NULL) { + utf8 = translate_into_utf8(str, tok->enc); + if (utf8 == NULL) + return error_ret(tok); + str = PyBytes_AsString(utf8); + } + for (s = str;; s++) { + if (*s == '\0') break; + else if (*s == '\n') { + assert(lineno < 2); + newl[lineno] = s; + lineno++; + if (lineno == 2) break; + } + } + tok->enc = NULL; + /* need to check line 1 and 2 separately since check_coding_spec + assumes a single line as input */ + if (newl[0]) { + if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { + return NULL; + } + if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) { + if (!check_coding_spec(newl[0]+1, newl[1] - newl[0], + tok, buf_setreadl)) + return NULL; + } + } + if (tok->enc != NULL) { + assert(utf8 == NULL); + utf8 = translate_into_utf8(str, tok->enc); + if (utf8 == NULL) + return error_ret(tok); + str = PyBytes_AS_STRING(utf8); + } + assert(tok->decoding_buffer == NULL); + tok->decoding_buffer = utf8; /* CAUTION */ + return str; +} + +/* Set up tokenizer for string */ +struct tok_state * +_PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf) +{ + struct tok_state *tok = tok_new(); + char *decoded; + + if (tok == NULL) + return NULL; + decoded = decode_str(str, exec_input, tok, preserve_crlf); + if (decoded == NULL) { + _PyTokenizer_Free(tok); + return NULL; + } + + tok->buf = tok->cur = tok->inp = decoded; + tok->end = decoded; + tok->underflow = &tok_underflow_string; + return tok; +} \ No newline at end of file diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h new file mode 100644 index 00000000000000..e3a0a7e67e40a0 --- /dev/null +++ b/Parser/tokenizer/tokenizer.h @@ -0,0 +1,14 @@ +#ifndef Py_TOKENIZER_H +#define Py_TOKENIZER_H + +#include "Python.h" + +extern struct tok_state *_PyTokenizer_FromString(const char *, int, int); +extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int); +extern struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int); +extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*, + const char *, const char *); + +#define tok_dump _Py_tok_dump + +#endif /* !Py_TOKENIZER_H */ diff --git a/Parser/tokenizer/utf8_tokenizer.c b/Parser/tokenizer/utf8_tokenizer.c new file mode 100644 index 00000000000000..412ac85a30f459 --- /dev/null +++ b/Parser/tokenizer/utf8_tokenizer.c @@ -0,0 +1,55 @@ +#include "Python.h" +#include "errcode.h" + +#include "helpers.h" +#include "../lexer/state.h" + +static int +tok_underflow_string(struct tok_state *tok) { + char *end = strchr(tok->inp, '\n'); + if (end != NULL) { + end++; + } + else { + end = strchr(tok->inp, '\0'); + if (end == tok->inp) { + tok->done = E_EOF; + return 0; + } + } + if (tok->start == NULL) { + tok->buf = tok->cur; + } + tok->line_start = tok->cur; + ADVANCE_LINENO(); + tok->inp = end; + return 1; +} + +/* Set up tokenizer for UTF-8 string */ +struct tok_state * +_PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) +{ + struct tok_state *tok = tok_new(); + char *translated; + if (tok == NULL) + return NULL; + tok->input = translated = translate_newlines(str, exec_input, preserve_crlf, tok); + if (translated == NULL) { + _PyTokenizer_Free(tok); + return NULL; + } + tok->decoding_state = STATE_NORMAL; + tok->enc = NULL; + tok->str = translated; + tok->encoding = new_string("utf-8", 5, tok); + if (!tok->encoding) { + _PyTokenizer_Free(tok); + return NULL; + } + + tok->buf = tok->cur = tok->inp = translated; + tok->end = translated; + tok->underflow = &tok_underflow_string; + return tok; +} diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c index 1b021069c5e10b..83b4aa4b1a7e11 100644 --- a/Python/Python-tokenize.c +++ b/Python/Python-tokenize.c @@ -1,6 +1,8 @@ #include "Python.h" #include "errcode.h" -#include "../Parser/tokenizer.h" +#include "../Parser/lexer/state.h" +#include "../Parser/lexer/lexer.h" +#include "../Parser/tokenizer/tokenizer.h" #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() #include "../Parser/pegen.h" // _PyPegen_byte_offset_to_character_offset() From 39257a03f73d1d44d78444dc03589d7417c49244 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 12:13:56 +0200 Subject: [PATCH 02/12] Fix build - remove extern's and fix tok_new decl --- Parser/lexer/lexer.h | 2 +- Parser/lexer/state.h | 8 ++++---- Parser/tokenizer/tokenizer.h | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index 154051ce0c3bc1..f2772452c3c7a5 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -5,6 +5,6 @@ int update_fstring_expr(struct tok_state *tok, char cur); -extern int _PyTokenizer_Get(struct tok_state *, struct token *); +int _PyTokenizer_Get(struct tok_state *, struct token *); #endif diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 606839635f02cd..d7146abe1d3b99 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -132,10 +132,10 @@ int type_comment_token_setup(struct tok_state *tok, struct token *token, int typ int end_col_offset, const char *start, const char *end); int token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); -extern struct tok_state *tok_new(); -extern void _PyTokenizer_Free(struct tok_state *); -extern void _PyToken_Free(struct token *); -extern void _PyToken_Init(struct token *); +struct tok_state *tok_new(void); +void _PyTokenizer_Free(struct tok_state *); +void _PyToken_Free(struct token *); +void _PyToken_Init(struct token *); #endif diff --git a/Parser/tokenizer/tokenizer.h b/Parser/tokenizer/tokenizer.h index e3a0a7e67e40a0..8fbeb2d6ae6df1 100644 --- a/Parser/tokenizer/tokenizer.h +++ b/Parser/tokenizer/tokenizer.h @@ -3,10 +3,10 @@ #include "Python.h" -extern struct tok_state *_PyTokenizer_FromString(const char *, int, int); -extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int); -extern struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int); -extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*, +struct tok_state *_PyTokenizer_FromString(const char *, int, int); +struct tok_state *_PyTokenizer_FromUTF8(const char *, int, int); +struct tok_state *_PyTokenizer_FromReadline(PyObject*, const char*, int, int); +struct tok_state *_PyTokenizer_FromFile(FILE *, const char*, const char *, const char *); #define tok_dump _Py_tok_dump From a4190a9a8678f7a2b9afd60fbd8e5df53a2cd6a6 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 12:41:32 +0200 Subject: [PATCH 03/12] Renames to remove smelly symbols --- Parser/lexer/buffer.c | 10 ++-- Parser/lexer/buffer.h | 6 +- Parser/lexer/lexer.c | 80 +++++++++++++-------------- Parser/lexer/lexer.h | 2 +- Parser/lexer/state.c | 6 +- Parser/lexer/state.h | 6 +- Parser/tokenizer/file_tokenizer.c | 42 +++++++------- Parser/tokenizer/helpers.c | 42 +++++++------- Parser/tokenizer/helpers.h | 32 +++++------ Parser/tokenizer/readline_tokenizer.c | 22 ++++---- Parser/tokenizer/string_tokenizer.c | 20 +++---- Parser/tokenizer/utf8_tokenizer.c | 6 +- 12 files changed, 137 insertions(+), 137 deletions(-) diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index 36c543b5738a51..f6502bf8f7f2d1 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -6,7 +6,7 @@ /* Traverse and remember all f-string buffers, in order to be able to restore them after reallocating tok->buf */ void -remember_fstring_buffers(struct tok_state *tok) +_PyLexer_remember_fstring_buffers(struct tok_state *tok) { int index; tokenizer_mode *mode; @@ -20,7 +20,7 @@ remember_fstring_buffers(struct tok_state *tok) /* Traverse and restore all f-string buffers after reallocating tok->buf */ void -restore_fstring_buffers(struct tok_state *tok) +_PyLexer_restore_fstring_buffers(struct tok_state *tok) { int index; tokenizer_mode *mode; @@ -47,7 +47,7 @@ restore_fstring_buffers(struct tok_state *tok) reached): see tok_nextc and its calls to tok_reserve_buf. */ int -tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) +_PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) { Py_ssize_t cur = tok->cur - tok->buf; Py_ssize_t oldsize = tok->inp - tok->buf; @@ -57,7 +57,7 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) Py_ssize_t start = tok->start == NULL ? -1 : tok->start - tok->buf; Py_ssize_t line_start = tok->start == NULL ? -1 : tok->line_start - tok->buf; Py_ssize_t multi_line_start = tok->multi_line_start - tok->buf; - remember_fstring_buffers(tok); + _PyLexer_remember_fstring_buffers(tok); newbuf = (char *)PyMem_Realloc(newbuf, newsize); if (newbuf == NULL) { tok->done = E_NOMEM; @@ -70,7 +70,7 @@ tok_reserve_buf(struct tok_state *tok, Py_ssize_t size) tok->start = start < 0 ? NULL : tok->buf + start; tok->line_start = line_start < 0 ? NULL : tok->buf + line_start; tok->multi_line_start = multi_line_start < 0 ? NULL : tok->buf + multi_line_start; - restore_fstring_buffers(tok); + _PyLexer_restore_fstring_buffers(tok); } return 1; } diff --git a/Parser/lexer/buffer.h b/Parser/lexer/buffer.h index 0e4cd9d70c4d63..bb218162ff4845 100644 --- a/Parser/lexer/buffer.h +++ b/Parser/lexer/buffer.h @@ -3,8 +3,8 @@ #include "pyport.h" -void remember_fstring_buffers(struct tok_state *tok); -void restore_fstring_buffers(struct tok_state *tok); -int tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); +void _PyLexer_remember_fstring_buffers(struct tok_state *tok); +void _PyLexer_restore_fstring_buffers(struct tok_state *tok); +int _PyLexer_tok_reserve_buf(struct tok_state *tok, Py_ssize_t size); #endif diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index dc58d9c0ce9934..c7134ab868bfbd 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -38,9 +38,9 @@ static inline tokenizer_mode* TOK_NEXT_MODE(struct tok_state* tok) { #define TOK_NEXT_MODE(tok) (&(tok->tok_mode_stack[++tok->tok_mode_stack_index])) #endif -#define MAKE_TOKEN(token_type) token_setup(tok, token, token_type, p_start, p_end) +#define MAKE_TOKEN(token_type) _PyLexer_token_setup(tok, token, token_type, p_start, p_end) #define MAKE_TYPE_COMMENT_TOKEN(token_type, col_offset, end_col_offset) (\ - type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) + _PyLexer_type_comment_token_setup(tok, token, token_type, col_offset, end_col_offset, p_start, p_end)) /* Spaces in this constant are treated as "zero or more spaces or tabs" when tokenizing. */ @@ -69,7 +69,7 @@ tok_nextc(struct tok_state *tok) #if defined(Py_DEBUG) if (tok->debug) { fprintf(stderr, "line[%d] = ", tok->lineno); - print_escape(stderr, tok->cur, tok->inp - tok->cur); + _PyTokenizer_print_escape(stderr, tok->cur, tok->inp - tok->cur); fprintf(stderr, " tok->done = %d\n", tok->done); } #endif @@ -80,7 +80,7 @@ tok_nextc(struct tok_state *tok) tok->line_start = tok->cur; if (contains_null_bytes(tok->line_start, tok->inp - tok->line_start)) { - syntaxerror(tok, "source code cannot contain null bytes"); + _PyTokenizer_syntaxerror(tok, "source code cannot contain null bytes"); tok->cur = tok->inp; return EOF; } @@ -126,7 +126,7 @@ set_fstring_expr(struct tok_state* tok, struct token *token, char c) { } int -update_fstring_expr(struct tok_state *tok, char cur) +_PyLexer_update_fstring_expr(struct tok_state *tok, char cur) { assert(tok->cur != NULL); @@ -243,7 +243,7 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { } if (r) { tok_backup(tok, c); - if (parser_warn(tok, PyExc_SyntaxWarning, + if (_PyTokenizer_parser_warn(tok, PyExc_SyntaxWarning, "invalid %s literal", kind)) { return 0; @@ -253,7 +253,7 @@ verify_end_of_number(struct tok_state *tok, int c, const char *kind) { else /* In future releases, only error will remain. */ if (c < 128 && is_potential_identifier_char(c)) { tok_backup(tok, c); - syntaxerror(tok, "invalid %s literal", kind); + _PyTokenizer_syntaxerror(tok, "invalid %s literal", kind); return 0; } return 1; @@ -304,10 +304,10 @@ verify_identifier(struct tok_state *tok) } Py_DECREF(s); if (Py_UNICODE_ISPRINTABLE(ch)) { - syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); + _PyTokenizer_syntaxerror(tok, "invalid character '%c' (U+%04X)", ch, ch); } else { - syntaxerror(tok, "invalid non-printable character U+%04X", ch); + _PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", ch); } return 0; } @@ -330,7 +330,7 @@ tok_decimal_tail(struct tok_state *tok) c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { tok_backup(tok, c); - syntaxerror(tok, "invalid decimal literal"); + _PyTokenizer_syntaxerror(tok, "invalid decimal literal"); return 0; } } @@ -432,7 +432,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t if (col == tok->indstack[tok->indent]) { /* No change */ if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(indenterror(tok)); + return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); } } else if (col > tok->indstack[tok->indent]) { @@ -443,7 +443,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } if (altcol <= tok->altindstack[tok->indent]) { - return MAKE_TOKEN(indenterror(tok)); + return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); } tok->pendin++; tok->indstack[++tok->indent] = col; @@ -462,7 +462,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } if (altcol != tok->altindstack[tok->indent]) { - return MAKE_TOKEN(indenterror(tok)); + return MAKE_TOKEN(_PyTokenizer_indenterror(tok)); } } } @@ -714,7 +714,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (!Py_ISXDIGIT(c)) { tok_backup(tok, c); - return MAKE_TOKEN(syntaxerror(tok, "invalid hexadecimal literal")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid hexadecimal literal")); } do { c = tok_nextc(tok); @@ -733,12 +733,12 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (c < '0' || c >= '8') { if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(syntaxerror(tok, + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in octal literal", c)); } else { tok_backup(tok, c); - return MAKE_TOKEN(syntaxerror(tok, "invalid octal literal")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid octal literal")); } } do { @@ -746,7 +746,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } while ('0' <= c && c < '8'); } while (c == '_'); if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(syntaxerror(tok, + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in octal literal", c)); } if (!verify_end_of_number(tok, c, "octal")) { @@ -762,11 +762,11 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (c != '0' && c != '1') { if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(syntaxerror(tok, "invalid digit '%c' in binary literal", c)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); } else { tok_backup(tok, c); - return MAKE_TOKEN(syntaxerror(tok, "invalid binary literal")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid binary literal")); } } do { @@ -774,7 +774,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } while (c == '0' || c == '1'); } while (c == '_'); if (Py_ISDIGIT(c)) { - return MAKE_TOKEN(syntaxerror(tok, "invalid digit '%c' in binary literal", c)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid digit '%c' in binary literal", c)); } if (!verify_end_of_number(tok, c, "binary")) { return MAKE_TOKEN(ERRORTOKEN); @@ -789,7 +789,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { tok_backup(tok, c); - return MAKE_TOKEN(syntaxerror(tok, "invalid decimal literal")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); } } if (c != '0') { @@ -818,7 +818,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t else if (nonzero && !tok->tok_extra_tokens) { /* Old-style octal: now disallowed. */ tok_backup(tok, c); - return MAKE_TOKEN(syntaxerror_known_range( + return MAKE_TOKEN(_PyTokenizer_syntaxerror_known_range( tok, (int)(tok->start + 1 - tok->line_start), (int)(zeros_end - tok->line_start), "leading zeros in decimal integer " @@ -859,7 +859,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t c = tok_nextc(tok); if (!Py_ISDIGIT(c)) { tok_backup(tok, c); - return MAKE_TOKEN(syntaxerror(tok, "invalid decimal literal")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid decimal literal")); } } else if (!Py_ISDIGIT(c)) { tok_backup(tok, c); @@ -928,7 +928,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t p_start = tok->start; p_end = tok->cur; if (tok->tok_mode_stack_index + 1 >= MAXFSTRINGLEVEL) { - return MAKE_TOKEN(syntaxerror(tok, "too many nested f-strings")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested f-strings")); } tokenizer_mode *the_current_tok = TOK_NEXT_MODE(tok); the_current_tok->kind = TOK_FSTRING_MODE; @@ -1020,12 +1020,12 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t tokenizer_mode *the_current_tok = TOK_GET_MODE(tok); if (the_current_tok->f_string_quote == quote && the_current_tok->f_string_quote_size == quote_size) { - return MAKE_TOKEN(syntaxerror(tok, "f-string: expecting '}'", start)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expecting '}'", start)); } } if (quote_size == 3) { - syntaxerror(tok, "unterminated triple-quoted string literal" + _PyTokenizer_syntaxerror(tok, "unterminated triple-quoted string literal" " (detected at line %d)", start); if (c != '\n') { tok->done = E_EOFS; @@ -1033,7 +1033,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } else { - syntaxerror(tok, "unterminated string literal (detected at" + _PyTokenizer_syntaxerror(tok, "unterminated string literal (detected at" " line %d)", start); if (c != '\n') { tok->done = E_EOLS; @@ -1076,7 +1076,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t * by the `{` case, so for ensuring that we are on the 0th level, we need * to adjust it manually */ int cursor = current_tok->curly_bracket_depth - (c != '{'); - if (cursor == 0 && !update_fstring_expr(tok, c)) { + if (cursor == 0 && !_PyLexer_update_fstring_expr(tok, c)) { return MAKE_TOKEN(ENDMARKER); } if (cursor == 0 && c != '{' && set_fstring_expr(tok, token, c)) { @@ -1117,7 +1117,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t case '[': case '{': if (tok->level >= MAXLEVEL) { - return MAKE_TOKEN(syntaxerror(tok, "too many nested parentheses")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "too many nested parentheses")); } tok->parenstack[tok->level] = c; tok->parenlinenostack[tok->level] = tok->lineno; @@ -1131,10 +1131,10 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t case ']': case '}': if (INSIDE_FSTRING(tok) && !current_tok->curly_bracket_depth && c == '}') { - return MAKE_TOKEN(syntaxerror(tok, "f-string: single '}' is not allowed")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: single '}' is not allowed")); } if (!tok->tok_extra_tokens && !tok->level) { - return MAKE_TOKEN(syntaxerror(tok, "unmatched '%c'", c)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unmatched '%c'", c)); } if (tok->level > 0) { tok->level--; @@ -1151,17 +1151,17 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t assert(current_tok->curly_bracket_depth >= 0); int previous_bracket = current_tok->curly_bracket_depth - 1; if (previous_bracket == current_tok->curly_bracket_expr_start_depth) { - return MAKE_TOKEN(syntaxerror(tok, "f-string: unmatched '%c'", c)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: unmatched '%c'", c)); } } if (tok->parenlinenostack[tok->level] != tok->lineno) { - return MAKE_TOKEN(syntaxerror(tok, + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "closing parenthesis '%c' does not match " "opening parenthesis '%c' on line %d", c, opening, tok->parenlinenostack[tok->level])); } else { - return MAKE_TOKEN(syntaxerror(tok, + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "closing parenthesis '%c' does not match " "opening parenthesis '%c'", c, opening)); @@ -1183,7 +1183,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t } if (!Py_UNICODE_ISPRINTABLE(c)) { - return MAKE_TOKEN(syntaxerror(tok, "invalid non-printable character U+%04X", c)); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c)); } if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) { @@ -1218,7 +1218,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct if (peek1 != '{') { current_tok->curly_bracket_expr_start_depth++; if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { - return MAKE_TOKEN(syntaxerror(tok, "f-string: expressions nested too deeply")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expressions nested too deeply")); } TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; return tok_get_normal_mode(tok, current_tok, token); @@ -1294,12 +1294,12 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok->lineno = the_current_tok->f_string_line_start; if (current_tok->f_string_quote_size == 3) { - return MAKE_TOKEN(syntaxerror(tok, + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unterminated triple-quoted f-string literal" " (detected at line %d)", start)); } else { - return MAKE_TOKEN(syntaxerror(tok, + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "unterminated f-string literal (detected at" " line %d)", start)); } @@ -1319,7 +1319,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct tok_backup(tok, c); current_tok->curly_bracket_expr_start_depth++; if (current_tok->curly_bracket_expr_start_depth >= MAX_EXPR_NESTING) { - return MAKE_TOKEN(syntaxerror(tok, "f-string: expressions nested too deeply")); + return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "f-string: expressions nested too deeply")); } TOK_GET_MODE(tok)->kind = TOK_REGULAR_MODE; p_start = tok->start; @@ -1362,7 +1362,7 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct // to the loop for the next iteration. if (peek == '{' || peek == '}') { if (!current_tok->f_string_raw) { - if (warn_invalid_escape_sequence(tok, peek)) { + if (_PyTokenizer_warn_invalid_escape_sequence(tok, peek)) { return MAKE_TOKEN(ERRORTOKEN); } } diff --git a/Parser/lexer/lexer.h b/Parser/lexer/lexer.h index f2772452c3c7a5..7f21bf56bba2d1 100644 --- a/Parser/lexer/lexer.h +++ b/Parser/lexer/lexer.h @@ -3,7 +3,7 @@ #include "state.h" -int update_fstring_expr(struct tok_state *tok, char cur); +int _PyLexer_update_fstring_expr(struct tok_state *tok, char cur); int _PyTokenizer_Get(struct tok_state *, struct token *); diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 809f47f21d1ad6..653ddafd411095 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -10,7 +10,7 @@ /* Create and initialize a new tok_state structure */ struct tok_state * -tok_new(void) +_PyTokenizer_tok_new(void) { struct tok_state *tok = (struct tok_state *)PyMem_Malloc( sizeof(struct tok_state)); @@ -113,7 +113,7 @@ _PyToken_Init(struct token *token) { } int -type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, +_PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, int end_col_offset, const char *start, const char *end) { token->level = tok->level; @@ -126,7 +126,7 @@ type_comment_token_setup(struct tok_state *tok, struct token *token, int type, i } int -token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) +_PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end) { assert((start == NULL && end == NULL) || (start != NULL && end != NULL)); token->level = tok->level; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index d7146abe1d3b99..61d090d6d2fe21 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -128,11 +128,11 @@ struct tok_state { #endif }; -int type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, +int _PyLexer_type_comment_token_setup(struct tok_state *tok, struct token *token, int type, int col_offset, int end_col_offset, const char *start, const char *end); -int token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); +int _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, const char *start, const char *end); -struct tok_state *tok_new(void); +struct tok_state *_PyTokenizer_tok_new(void); void _PyTokenizer_Free(struct tok_state *); void _PyToken_Free(struct token *); void _PyToken_Init(struct token *); diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 535790cdb38fbf..05ab5a02f53bdd 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -56,7 +56,7 @@ static int tok_readline_raw(struct tok_state *tok) { do { - if (!tok_reserve_buf(tok, BUFSIZ)) { + if (!_PyLexer_tok_reserve_buf(tok, BUFSIZ)) { return 0; } int n_chars = (int)(tok->end - tok->inp); @@ -86,7 +86,7 @@ tok_readline_recode(struct tok_state *tok) { if (line == NULL) { line = PyObject_CallNoArgs(tok->decoding_readline); if (line == NULL) { - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } } @@ -95,14 +95,14 @@ tok_readline_recode(struct tok_state *tok) { } buf = PyUnicode_AsUTF8AndSize(line, &buflen); if (buf == NULL) { - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } // Make room for the null terminator *and* potentially // an extra newline character that we may need to artificially // add. size_t buffer_size = buflen + 2; - if (!tok_reserve_buf(tok, buffer_size)) { + if (!_PyLexer_tok_reserve_buf(tok, buffer_size)) { goto error; } memcpy(tok->inp, buf, buflen); @@ -132,7 +132,7 @@ static void fp_ungetc(int c, struct tok_state *tok) { /* Set the readline function for TOK to a StreamReader's readline function. The StreamReader is named ENC. - This function is called from check_bom and check_coding_spec. + This function is called from _PyTokenizer_check_bom and _PyTokenizer_check_coding_spec. ENC is usually identical to the future value of tok->encoding, except for the (currently unsupported) case of UTF-16. @@ -195,7 +195,7 @@ tok_underflow_interactive(struct tok_state *tok) { } char *newtok = PyOS_Readline(tok->fp ? tok->fp : stdin, stdout, tok->prompt); if (newtok != NULL) { - char *translated = translate_newlines(newtok, 0, 0, tok); + char *translated = _PyTokenizer_translate_newlines(newtok, 0, 0, tok); PyMem_Free(newtok); if (translated == NULL) { return 0; @@ -206,7 +206,7 @@ tok_underflow_interactive(struct tok_state *tok) { /* Recode to UTF-8 */ Py_ssize_t buflen; const char* buf; - PyObject *u = translate_into_utf8(newtok, tok->encoding); + PyObject *u = _PyTokenizer_translate_into_utf8(newtok, tok->encoding); PyMem_Free(newtok); if (u == NULL) { tok->done = E_DECODE; @@ -240,10 +240,10 @@ tok_underflow_interactive(struct tok_state *tok) { } else if (tok->start != NULL) { Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf; - remember_fstring_buffers(tok); + _PyLexer_remember_fstring_buffers(tok); size_t size = strlen(newtok); ADVANCE_LINENO(); - if (!tok_reserve_buf(tok, size + 1)) { + if (!_PyLexer_tok_reserve_buf(tok, size + 1)) { PyMem_Free(tok->buf); tok->buf = NULL; PyMem_Free(newtok); @@ -253,10 +253,10 @@ tok_underflow_interactive(struct tok_state *tok) { PyMem_Free(newtok); tok->inp += size; tok->multi_line_start = tok->buf + cur_multi_line_start; - restore_fstring_buffers(tok); + _PyLexer_restore_fstring_buffers(tok); } else { - remember_fstring_buffers(tok); + _PyLexer_remember_fstring_buffers(tok); ADVANCE_LINENO(); PyMem_Free(tok->buf); tok->buf = newtok; @@ -264,7 +264,7 @@ tok_underflow_interactive(struct tok_state *tok) { tok->line_start = tok->buf; tok->inp = strchr(tok->buf, '\0'); tok->end = tok->inp + 1; - restore_fstring_buffers(tok); + _PyLexer_restore_fstring_buffers(tok); } if (tok->done != E_OK) { if (tok->prompt != NULL) { @@ -273,7 +273,7 @@ tok_underflow_interactive(struct tok_state *tok) { return 0; } - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) { return 0; } return 1; @@ -288,8 +288,8 @@ tok_underflow_file(struct tok_state *tok) { /* We have not yet determined the encoding. If an encoding is found, use the file-pointer reader functions from now on. */ - if (!check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) { - error_ret(tok); + if (!_PyTokenizer_check_bom(fp_getc, fp_ungetc, fp_setreadl, tok)) { + _PyTokenizer_error_ret(tok); return 0; } assert(tok->decoding_state != STATE_INIT); @@ -320,7 +320,7 @@ tok_underflow_file(struct tok_state *tok) { tok->implicit_newline = 1; } - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) { return 0; } @@ -329,7 +329,7 @@ tok_underflow_file(struct tok_state *tok) { if (tok->lineno > 2) { tok->decoding_state = STATE_NORMAL; } - else if (!check_coding_spec(tok->cur, strlen(tok->cur), + else if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), tok, fp_setreadl)) { return 0; @@ -337,8 +337,8 @@ tok_underflow_file(struct tok_state *tok) { } /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ - if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { - error_ret(tok); + if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { + _PyTokenizer_error_ret(tok); return 0; } assert(tok->done == E_OK); @@ -350,7 +350,7 @@ struct tok_state * _PyTokenizer_FromFile(FILE *fp, const char* enc, const char *ps1, const char *ps2) { - struct tok_state *tok = tok_new(); + struct tok_state *tok = _PyTokenizer_tok_new(); if (tok == NULL) return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { @@ -370,7 +370,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, if (enc != NULL) { /* Must copy encoding declaration since it gets copied into the parse tree. */ - tok->encoding = new_string(enc, strlen(enc), tok); + tok->encoding = _PyTokenizer_new_string(enc, strlen(enc), tok); if (!tok->encoding) { _PyTokenizer_Free(tok); return NULL; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 4a51c44576b13b..c52b76a65a524f 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -61,7 +61,7 @@ _syntaxerror_range(struct tok_state *tok, const char *format, } int -syntaxerror(struct tok_state *tok, const char *format, ...) +_PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...) { // This errors are cleaned on startup. Todo: Fix it. va_list vargs; @@ -72,7 +72,7 @@ syntaxerror(struct tok_state *tok, const char *format, ...) } int -syntaxerror_known_range(struct tok_state *tok, +_PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...) { @@ -84,7 +84,7 @@ syntaxerror_known_range(struct tok_state *tok, } int -indenterror(struct tok_state *tok) +_PyTokenizer_indenterror(struct tok_state *tok) { tok->done = E_TABSPACE; tok->cur = tok->inp; @@ -92,7 +92,7 @@ indenterror(struct tok_state *tok) } char * -error_ret(struct tok_state *tok) /* XXX */ +_PyTokenizer_error_ret(struct tok_state *tok) /* XXX */ { tok->decoding_erred = 1; if ((tok->fp != NULL || tok->readline != NULL) && tok->buf != NULL) {/* see _PyTokenizer_Free */ @@ -106,7 +106,7 @@ error_ret(struct tok_state *tok) /* XXX */ } int -warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) +_PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char) { if (!tok->report_warnings) { return 0; @@ -129,7 +129,7 @@ warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_cha /* Replace the SyntaxWarning exception with a SyntaxError to get a more accurate error report */ PyErr_Clear(); - return syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char); + return _PyTokenizer_syntaxerror(tok, "invalid escape sequence '\\%c'", (char) first_invalid_escape_char); } return -1; @@ -140,7 +140,7 @@ warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_cha } int -parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) +_PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { if (!tok->report_warnings) { return 0; @@ -161,7 +161,7 @@ parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) /* Replace the DeprecationWarning exception with a SyntaxError to get a more accurate error report */ PyErr_Clear(); - syntaxerror(tok, "%U", errmsg); + _PyTokenizer_syntaxerror(tok, "%U", errmsg); } goto error; } @@ -178,7 +178,7 @@ parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) /* ############## STRING MANIPULATION ############## */ char * -new_string(const char *s, Py_ssize_t len, struct tok_state *tok) +_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok) { char* result = (char *)PyMem_Malloc(len + 1); if (!result) { @@ -191,7 +191,7 @@ new_string(const char *s, Py_ssize_t len, struct tok_state *tok) } PyObject * -translate_into_utf8(const char* str, const char* enc) { +_PyTokenizer_translate_into_utf8(const char* str, const char* enc) { PyObject *utf8; PyObject* buf = PyUnicode_Decode(str, strlen(str), enc, NULL); if (buf == NULL) @@ -202,7 +202,7 @@ translate_into_utf8(const char* str, const char* enc) { } char * -translate_newlines(const char *s, int exec_input, int preserve_crlf, +_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok) { int skip_next_lf = 0; size_t needed_length = strlen(s) + 2, final_length; @@ -255,7 +255,7 @@ translate_newlines(const char *s, int exec_input, int preserve_crlf, invoke the set_readline function with the new encoding. Return 1 on success, 0 on failure. */ int -check_bom(int get_char(struct tok_state *), +_PyTokenizer_check_bom(int get_char(struct tok_state *), void unget_char(int, struct tok_state *), int set_readline(struct tok_state *, const char *), struct tok_state *tok) @@ -285,7 +285,7 @@ check_bom(int get_char(struct tok_state *), } if (tok->encoding != NULL) PyMem_Free(tok->encoding); - tok->encoding = new_string("utf-8", 5, tok); + tok->encoding = _PyTokenizer_new_string("utf-8", 5, tok); if (!tok->encoding) return 0; /* No need to set_readline: input is already utf-8 */ @@ -352,14 +352,14 @@ get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *t t++; if (begin < t) { - char* r = new_string(begin, t - begin, tok); + char* r = _PyTokenizer_new_string(begin, t - begin, tok); const char* q; if (!r) return 0; q = get_normal_name(r); if (r != q) { PyMem_Free(r); - r = new_string(q, strlen(q), tok); + r = _PyTokenizer_new_string(q, strlen(q), tok); if (!r) return 0; } @@ -376,7 +376,7 @@ get_coding_spec(const char *s, char **spec, Py_ssize_t size, struct tok_state *t This function receives the tok_state and the new encoding. Return 1 on success, 0 on failure. */ int -check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, +_PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, int set_readline(struct tok_state *, const char *)) { char *cs; @@ -406,7 +406,7 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, if (tok->encoding == NULL) { assert(tok->decoding_readline == NULL); if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) { - error_ret(tok); + _PyTokenizer_error_ret(tok); PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); PyMem_Free(cs); return 0; @@ -414,7 +414,7 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, tok->encoding = cs; } else { /* then, compare cs with BOM */ if (strcmp(tok->encoding, cs) != 0) { - error_ret(tok); + _PyTokenizer_error_ret(tok); PyErr_Format(PyExc_SyntaxError, "encoding problem: %s with BOM", cs); PyMem_Free(cs); @@ -488,7 +488,7 @@ valid_utf8(const unsigned char* s) } int -ensure_utf8(char *line, struct tok_state *tok) +_PyTokenizer_ensure_utf8(char *line, struct tok_state *tok) { int badchar = 0; unsigned char *c; @@ -516,7 +516,7 @@ ensure_utf8(char *line, struct tok_state *tok) #if defined(Py_DEBUG) void -print_escape(FILE *f, const char *s, Py_ssize_t size) +_PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size) { if (s == NULL) { fputs("NULL", f); @@ -543,7 +543,7 @@ print_escape(FILE *f, const char *s, Py_ssize_t size) } void -tok_dump(int type, char *start, char *end) +_PyTokenizer_tok_dump(int type, char *start, char *end) { fprintf(stderr, "%s", _PyParser_TokenNames[type]); if (type == NAME || type == NUMBER || type == STRING || type == OP) diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 7bb9fa15895500..ae36d62a4fd269 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -9,28 +9,28 @@ tok->lineno++; \ tok->col_offset = 0; -int syntaxerror(struct tok_state *tok, const char *format, ...); -int syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); -int indenterror(struct tok_state *tok); -int warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); -int parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); -char *error_ret(struct tok_state *tok); - -char *new_string(const char *s, Py_ssize_t len, struct tok_state *tok); -char *translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); -PyObject *translate_into_utf8(const char* str, const char* enc); - -int check_bom(int get_char(struct tok_state *), +int _PyTokenizer_syntaxerror(struct tok_state *tok, const char *format, ...); +int _PyTokenizer_syntaxerror_known_range(struct tok_state *tok, int col_offset, int end_col_offset, const char *format, ...); +int _PyTokenizer_indenterror(struct tok_state *tok); +int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); +int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); +char *_PyTokenizer_error_ret(struct tok_state *tok); + +char *_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok); +char *_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); +PyObject *_PyTokenizer_translate_into_utf8(const char* str, const char* enc); + +int _PyTokenizer_check_bom(int get_char(struct tok_state *), void unget_char(int, struct tok_state *), int set_readline(struct tok_state *, const char *), struct tok_state *tok); -int check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, +int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, int set_readline(struct tok_state *, const char *)); -int ensure_utf8(char *line, struct tok_state *tok); +int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok); #if Py_DEBUG -void print_escape(FILE *f, const char *s, Py_ssize_t size); -void tok_dump(int type, char *start, char *end); +void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); +void _PyTokenizer_tok_dump(int type, char *start, char *end); #endif diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 892fa4691506cb..3a2893b3e2f01e 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -15,26 +15,26 @@ tok_readline_string(struct tok_state* tok) { PyErr_Clear(); return 1; } - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } if(tok->encoding != NULL) { if (!PyBytes_Check(raw_line)) { PyErr_Format(PyExc_TypeError, "readline() returned a non-bytes object"); - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } line = PyUnicode_Decode(PyBytes_AS_STRING(raw_line), PyBytes_GET_SIZE(raw_line), tok->encoding, "replace"); Py_CLEAR(raw_line); if (line == NULL) { - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } } else { if(!PyUnicode_Check(raw_line)) { PyErr_Format(PyExc_TypeError, "readline() returned a non-string object"); - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } line = raw_line; @@ -43,7 +43,7 @@ tok_readline_string(struct tok_state* tok) { Py_ssize_t buflen; const char* buf = PyUnicode_AsUTF8AndSize(line, &buflen); if (buf == NULL) { - error_ret(tok); + _PyTokenizer_error_ret(tok); goto error; } @@ -51,7 +51,7 @@ tok_readline_string(struct tok_state* tok) { // an extra newline character that we may need to artificially // add. size_t buffer_size = buflen + 2; - if (!tok_reserve_buf(tok, buffer_size)) { + if (!_PyLexer_tok_reserve_buf(tok, buffer_size)) { goto error; } memcpy(tok->inp, buf, buflen); @@ -90,15 +90,15 @@ tok_underflow_readline(struct tok_state* tok) { tok->implicit_newline = 1; } - if (tok->tok_mode_stack_index && !update_fstring_expr(tok, 0)) { + if (tok->tok_mode_stack_index && !_PyLexer_update_fstring_expr(tok, 0)) { return 0; } ADVANCE_LINENO(); /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ - if (!tok->encoding && !ensure_utf8(tok->cur, tok)) { - error_ret(tok); + if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { + _PyTokenizer_error_ret(tok); return 0; } assert(tok->done == E_OK); @@ -109,7 +109,7 @@ struct tok_state * _PyTokenizer_FromReadline(PyObject* readline, const char* enc, int exec_input, int preserve_crlf) { - struct tok_state *tok = tok_new(); + struct tok_state *tok = _PyTokenizer_tok_new(); if (tok == NULL) return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { @@ -120,7 +120,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, tok->end = tok->buf + BUFSIZ; tok->fp = NULL; if (enc != NULL) { - tok->encoding = new_string(enc, strlen(enc), tok); + tok->encoding = _PyTokenizer_new_string(enc, strlen(enc), tok); if (!tok->encoding) { _PyTokenizer_Free(tok); return NULL; diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 7e48a568eb0c46..6459b8a0782e41 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -58,19 +58,19 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr const char *s; const char *newl[2] = {NULL, NULL}; int lineno = 0; - tok->input = str = translate_newlines(input, single, preserve_crlf, tok); + tok->input = str = _PyTokenizer_translate_newlines(input, single, preserve_crlf, tok); if (str == NULL) return NULL; tok->enc = NULL; tok->str = str; - if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) - return error_ret(tok); + if (!_PyTokenizer_check_bom(buf_getc, buf_ungetc, buf_setreadl, tok)) + return _PyTokenizer_error_ret(tok); str = tok->str; /* string after BOM if any */ assert(str); if (tok->enc != NULL) { - utf8 = translate_into_utf8(str, tok->enc); + utf8 = _PyTokenizer_translate_into_utf8(str, tok->enc); if (utf8 == NULL) - return error_ret(tok); + return _PyTokenizer_error_ret(tok); str = PyBytes_AsString(utf8); } for (s = str;; s++) { @@ -86,20 +86,20 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr /* need to check line 1 and 2 separately since check_coding_spec assumes a single line as input */ if (newl[0]) { - if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { + if (!_PyTokenizer_check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { return NULL; } if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) { - if (!check_coding_spec(newl[0]+1, newl[1] - newl[0], + if (!_PyTokenizer_check_coding_spec(newl[0]+1, newl[1] - newl[0], tok, buf_setreadl)) return NULL; } } if (tok->enc != NULL) { assert(utf8 == NULL); - utf8 = translate_into_utf8(str, tok->enc); + utf8 = _PyTokenizer_translate_into_utf8(str, tok->enc); if (utf8 == NULL) - return error_ret(tok); + return _PyTokenizer_error_ret(tok); str = PyBytes_AS_STRING(utf8); } assert(tok->decoding_buffer == NULL); @@ -111,7 +111,7 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr struct tok_state * _PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf) { - struct tok_state *tok = tok_new(); + struct tok_state *tok = _PyTokenizer_tok_new(); char *decoded; if (tok == NULL) diff --git a/Parser/tokenizer/utf8_tokenizer.c b/Parser/tokenizer/utf8_tokenizer.c index 412ac85a30f459..1a925f445400fa 100644 --- a/Parser/tokenizer/utf8_tokenizer.c +++ b/Parser/tokenizer/utf8_tokenizer.c @@ -30,11 +30,11 @@ tok_underflow_string(struct tok_state *tok) { struct tok_state * _PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) { - struct tok_state *tok = tok_new(); + struct tok_state *tok = _PyTokenizer_tok_new(); char *translated; if (tok == NULL) return NULL; - tok->input = translated = translate_newlines(str, exec_input, preserve_crlf, tok); + tok->input = translated = _PyTokenizer_translate_newlines(str, exec_input, preserve_crlf, tok); if (translated == NULL) { _PyTokenizer_Free(tok); return NULL; @@ -42,7 +42,7 @@ _PyTokenizer_FromUTF8(const char *str, int exec_input, int preserve_crlf) tok->decoding_state = STATE_NORMAL; tok->enc = NULL; tok->str = translated; - tok->encoding = new_string("utf-8", 5, tok); + tok->encoding = _PyTokenizer_new_string("utf-8", 5, tok); if (!tok->encoding) { _PyTokenizer_Free(tok); return NULL; From a949d15bb074e76b3310c335f99288c1001c3a5f Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 14:14:37 +0200 Subject: [PATCH 04/12] Fix build errors with subdirs --- configure | 2 ++ configure.ac | 2 ++ 2 files changed, 4 insertions(+) diff --git a/configure b/configure index d68c4e8e9b0756..c87f518382f2ec 100755 --- a/configure +++ b/configure @@ -26679,6 +26679,8 @@ SRCDIRS="\ Modules/expat \ Objects \ Parser \ + Parser/tokenizer \ + Parser/lexer \ Programs \ Python \ Python/frozen_modules \ diff --git a/configure.ac b/configure.ac index 9a9c76f3545c30..cd69f0ede54496 100644 --- a/configure.ac +++ b/configure.ac @@ -6526,6 +6526,8 @@ SRCDIRS="\ Modules/expat \ Objects \ Parser \ + Parser/tokenizer \ + Parser/lexer \ Programs \ Python \ Python/frozen_modules \ From 95662b8988f3e242c9bbc2f664c234dd4470638c Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 14:20:01 +0200 Subject: [PATCH 05/12] Fix windows build --- PCbuild/_freeze_module.vcxproj | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index bdcf29ba44dab5..aaa63fe9456fb7 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -172,7 +172,14 @@ - + + + + + + + + From ec41b56271cd6159c4781c4279ca1c9bc98e2495 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Wed, 11 Oct 2023 14:37:00 +0200 Subject: [PATCH 06/12] fixup! Fix windows build --- Makefile.pre.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile.pre.in b/Makefile.pre.in index 9755f3cfae7748..f70c1122c82a01 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1413,6 +1413,8 @@ regen-pegen-metaparser: .PHONY: regen-pegen regen-pegen: @$(MKDIR_P) $(srcdir)/Parser + @$(MKDIR_P) $(srcdir)/Parser/tokenizer + @$(MKDIR_P) $(srcdir)/Parser/lexer PYTHONPATH=$(srcdir)/Tools/peg_generator $(PYTHON_FOR_REGEN) -m pegen -q c \ $(srcdir)/Grammar/python.gram \ $(srcdir)/Grammar/Tokens \ From 815cb3476817b00b4b87adaba00e1803553fda0b Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 14:41:02 +0200 Subject: [PATCH 07/12] Fix ifdef --- Parser/tokenizer/helpers.c | 2 +- Parser/tokenizer/helpers.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index c52b76a65a524f..9c9d05bbef0f1a 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -514,7 +514,7 @@ _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok) /* ############## DEBUGGING STUFF ############## */ -#if defined(Py_DEBUG) +#ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size) { diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index ae36d62a4fd269..42ea13cd1f853f 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -28,7 +28,7 @@ int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok int set_readline(struct tok_state *, const char *)); int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok); -#if Py_DEBUG +#ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); void _PyTokenizer_tok_dump(int type, char *start, char *end); #endif From 20be8a0a97ff34474335175e235f4de1f11f776f Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 14:43:05 +0200 Subject: [PATCH 08/12] Add empty lines at the end --- Parser/tokenizer/readline_tokenizer.c | 2 +- Parser/tokenizer/string_tokenizer.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 3a2893b3e2f01e..a2637af9902dd3 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -131,4 +131,4 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, Py_INCREF(readline); tok->readline = readline; return tok; -} \ No newline at end of file +} diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 6459b8a0782e41..0c26d5df8d4a40 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -126,4 +126,4 @@ _PyTokenizer_FromString(const char *str, int exec_input, int preserve_crlf) tok->end = decoded; tok->underflow = &tok_underflow_string; return tok; -} \ No newline at end of file +} From 128fd0272967907a32299e47e4e5137e20a35cb5 Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 11 Oct 2023 12:48:08 +0000 Subject: [PATCH 09/12] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst new file mode 100644 index 00000000000000..6858bd6a9c73fc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst @@ -0,0 +1,3 @@ +Split the tokenizer into two separate directories: +- One part includes the actual lexeme producing logic and lives in `Parser/lexer`. +- The second part wraps the lexer according to the different tokenization modes we have (string, utf-8, file, interactive, readline) and lives in `Parser/tokenizer`. From df2c8cb9bfc86761fc06bdcd280cfd6ba8e59832 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 14:58:23 +0200 Subject: [PATCH 10/12] Fix lint errors --- .../2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst b/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst index 6858bd6a9c73fc..82ec9c1c30a74f 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2023-10-11-12-48-03.gh-issue-104169.bPoX8u.rst @@ -1,3 +1,4 @@ Split the tokenizer into two separate directories: -- One part includes the actual lexeme producing logic and lives in `Parser/lexer`. -- The second part wraps the lexer according to the different tokenization modes we have (string, utf-8, file, interactive, readline) and lives in `Parser/tokenizer`. +- One part includes the actual lexeme producing logic and lives in ``Parser/lexer``. +- The second part wraps the lexer according to the different tokenization modes + we have (string, utf-8, file, interactive, readline) and lives in ``Parser/tokenizer``. From 7e10dbf37eb72b57e107d48cd098e5037ae32144 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 15:29:20 +0200 Subject: [PATCH 11/12] Fix more windows stuff --- PCbuild/_freeze_module.vcxproj.filters | 23 +++++++++++++++- PCbuild/pythoncore.vcxproj | 15 +++++++++-- PCbuild/pythoncore.vcxproj.filters | 37 ++++++++++++++++++++++++-- 3 files changed, 70 insertions(+), 5 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index 45333fa97f1c64..279736b0fbbb13 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -397,7 +397,28 @@ Source Files - + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + Source Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 43a79fd5938486..f240779d2a5d4e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -362,7 +362,11 @@ - + + + + + @@ -507,7 +511,14 @@ - + + + + + + + + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index 59159ed609968b..6e13eb81018f9b 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -291,7 +291,19 @@ Objects - + + Parser + + + Parser + + + Parser + + + Parser + + Parser @@ -1139,7 +1151,28 @@ Parser - + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + + Parser + + Parser From aeeb5622be2b9d86b881628a9fa27e5a38cb45c5 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Wed, 11 Oct 2023 16:45:44 +0200 Subject: [PATCH 12/12] Fix c-analyzer --- Tools/c-analyzer/cpython/ignored.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index ca656f0760ab4f..0ba7ab15ce89b8 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -335,7 +335,7 @@ Objects/unicodeobject.c unicode_encode_call_errorhandler argparse - Objects/unicodeobject.c unicode_translate_call_errorhandler argparse - Parser/parser.c - reserved_keywords - Parser/parser.c - soft_keywords - -Parser/tokenizer.c - type_comment_prefix - +Parser/lexer/lexer.c - type_comment_prefix - Python/ast_opt.c fold_unaryop ops - Python/ceval.c - _PyEval_BinaryOps - Python/ceval.c - _Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS -