Skip to content

Commit abfbab6

Browse files
authored
gh-105718: Fix buffer allocation in tokenizer with readline (#105728)
1 parent d0f1afd commit abfbab6

File tree

3 files changed

+11
-5
lines changed

3 files changed

+11
-5
lines changed

Lib/test/test_tokenize.py

+10
Original file line numberDiff line numberDiff line change
@@ -2239,6 +2239,16 @@ def test_string(self):
22392239
FSTRING_START \'f"\' (1, 0) (1, 2)
22402240
FSTRING_MIDDLE 'abc\\\\\\ndef' (1, 2) (2, 3)
22412241
FSTRING_END '"' (2, 3) (2, 4)
2242+
""")
2243+
2244+
self.check_tokenize('''\
2245+
f"{
2246+
a}"''', """\
2247+
FSTRING_START 'f"' (1, 0) (1, 2)
2248+
LBRACE '{' (1, 2) (1, 3)
2249+
NAME 'a' (2, 0) (2, 1)
2250+
RBRACE '}' (2, 1) (2, 2)
2251+
FSTRING_END '"' (2, 2) (2, 3)
22422252
""")
22432253

22442254
self.check_tokenize(r'Rf"abc\

Parser/tokenizer.c

-4
Original file line numberDiff line numberDiff line change
@@ -1106,11 +1106,7 @@ tok_readline_string(struct tok_state* tok) {
11061106
tok->inp += buflen;
11071107
*tok->inp = '\0';
11081108

1109-
if (tok->start == NULL) {
1110-
tok->buf = tok->cur;
1111-
}
11121109
tok->line_start = tok->cur;
1113-
11141110
Py_DECREF(line);
11151111
return 1;
11161112
error:

Parser/tokenizer.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ typedef struct _tokenizer_mode {
6868
struct tok_state {
6969
/* Input state; buf <= cur <= inp <= end */
7070
/* NB an entire line is held in the buffer */
71-
char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL */
71+
char *buf; /* Input buffer, or NULL; malloc'ed if fp != NULL or readline != NULL */
7272
char *cur; /* Next character in buffer */
7373
char *inp; /* End of data in buffer */
7474
int fp_interactive; /* If the file descriptor is interactive */

0 commit comments

Comments
 (0)