bpo-39209: Manage correctly multi-line tokens in interactive mode

pablogsal · pablogsal · commit 13dadaa15044 · 2020-01-06T10:29:25.000Z
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-01-06-10-29-16.bpo-39209.QHAONe.rst
@@ -0,0 +1,2 @@
+Correctly handle multi-line tokens in interactive mode. Patch by Pablo
+Galindo.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok)
                 size_t start = tok->start - tok->buf;
                 size_t oldlen = tok->cur - tok->buf;
                 size_t newlen = oldlen + strlen(newtok);
+                Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;
                 char *buf = tok->buf;
                 buf = (char *)PyMem_REALLOC(buf, newlen+1);
                 tok->lineno++;
@@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok)
                 }
                 tok->buf = buf;
                 tok->cur = tok->buf + oldlen;
+                tok->multi_line_start = tok->buf + cur_multi_line_start;
                 tok->line_start = tok->cur;
                 strcpy(tok->buf + oldlen, newtok);
                 PyMem_FREE(newtok);

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Correctly handle multi-line tokens in interactive mode. Patch by Pablo`
	`2`	`+Galindo.`