bpo-46091: Correctly calculate indentation levels for whitespace lines with continuation characters

pablogsal · pablogsal · commit de9998a2cdb3 · 2021-12-16T00:32:35.000Z
diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
@@ -1076,8 +1076,7 @@ def test_literal_eval_malformed_lineno(self):
             ast.literal_eval(node)
 
     def test_literal_eval_syntax_errors(self):
-        msg = "unexpected character after line continuation character"
-        with self.assertRaisesRegex(SyntaxError, msg):
+        with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
             ast.literal_eval(r'''
                 \
                 (\
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
@@ -1507,6 +1507,27 @@ def test_empty_line_after_linecont(self):
         \
 
 pass
+"""
+        try:
+            compile(s, '<string>', 'exec')
+        except SyntaxError:
+            self.fail("Empty line after a line continuation character is valid.")
+
+        # See issue-46091
+        s = r"""\
+def fib(n):
+    \
+'''Print a Fibonacci series up to n.'''
+    \
+a, b = 0, 1
+    \
+while a < n:
+        \
+print(a, end=' ')
+        \
+a, b = b, a+b
+    \
+print()
 """
         try:
             compile(s, '<string>', 'exec')
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst b/Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst
@@ -0,0 +1,2 @@
+Correctly calculate indentation levels for lines with whitespace character
+that are ended by line continuation characters. Patch by Pablo Galindo
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -1347,6 +1347,24 @@ tok_decimal_tail(struct tok_state *tok)
 
 /* Get next token, after space stripping etc. */
 
+static inline int
+tok_continuation_line(struct tok_state *tok) {
+    int c = tok_nextc(tok);
+    if (c != '\n') {
+        tok->done = E_LINECONT;
+        return -1;
+    }
+    c = tok_nextc(tok);
+    if (c == EOF) {
+        tok->done = E_EOF;
+        tok->cur = tok->inp;
+        return -1;
+    } else {
+        tok_backup(tok, c);
+    }
+    return c;
+}
+
 static int
 tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
 {
@@ -1375,12 +1393,18 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
             else if (c == '\014')  {/* Control-L (formfeed) */
                 col = altcol = 0; /* For Emacs users */
             }
+            else if (c == '\\') {
+                c = tok_continuation_line(tok);
+                if (c == -1) {
+                    return ERRORTOKEN;
+                }
+            }
             else {
                 break;
             }
         }
         tok_backup(tok, c);
-        if (c == '#' || c == '\n' || c == '\\') {
+        if (c == '#' || c == '\n') {
             /* Lines with only whitespace and/or comments
                and/or a line continuation character
                shouldn't affect the indentation and are
@@ -1964,19 +1988,10 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
 
     /* Line continuation */
     if (c == '\\') {
-        c = tok_nextc(tok);
-        if (c != '\n') {
-            tok->done = E_LINECONT;
+        c = tok_continuation_line(tok);
+        if (c == -1) {
             return ERRORTOKEN;
         }
-        c = tok_nextc(tok);
-        if (c == EOF) {
-            tok->done = E_EOF;
-            tok->cur = tok->inp;
-            return ERRORTOKEN;
-        } else {
-            tok_backup(tok, c);
-        }
         tok->cont_line = 1;
         goto again; /* Read next line */
     }

Original file line number	Diff line number	Diff line change
`@@ -1076,8 +1076,7 @@ def test_literal_eval_malformed_lineno(self):`
`1076`	`1076`	`ast.literal_eval(node)`
`1077`	`1077`
`1078`	`1078`	`def test_literal_eval_syntax_errors(self):`
`1079`		`- msg = "unexpected character after line continuation character"`
`1080`		`- with self.assertRaisesRegex(SyntaxError, msg):`
	`1079`	`+ with self.assertRaisesRegex(SyntaxError, "unexpected indent"):`
`1081`	`1080`	`ast.literal_eval(r'''`
`1082`	`1081`	`\`
`1083`	`1082`	`(\`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+Correctly calculate indentation levels for lines with whitespace character`
	`2`	`+that are ended by line continuation characters. Patch by Pablo Galindo`