Skip to content

bpo-46091: Correctly calculate indentation levels for whitespace lines with continuation characters #30130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pablogsal merged 5 commits into from
Jan 25, 2022
Merged

bpo-46091: Correctly calculate indentation levels for whitespace lines with continuation characters #30130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
de9998a
bpo-46091: Correctly calculate indentation levels for whitespace line…
pablogsal Dec 16, 2021
118b66c
fixup! bpo-46091: Correctly calculate indentation levels for whitespa…
pablogsal Dec 16, 2021
c623e38
fixup! fixup! bpo-46091: Correctly calculate indentation levels for w…
pablogsal Dec 16, 2021
95b10a6
Fix comment
pablogsal Dec 16, 2021
61c95fe
fixup! fixup! fixup! bpo-46091: Correctly calculate indentation level…
pablogsal Dec 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Lib/test/test_ast.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1076,8 +1076,7 @@ def test_literal_eval_malformed_lineno(self):
ast.literal_eval(node)

def test_literal_eval_syntax_errors(self):
msg = "unexpected character after line continuation character"
with self.assertRaisesRegex(SyntaxError, msg):
with self.assertRaisesRegex(SyntaxError, "unexpected indent"):
ast.literal_eval(r'''
\
(\
Expand Down
30 changes: 30 additions & 0 deletions Lib/test/test_syntax.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1513,6 +1513,36 @@ def test_empty_line_after_linecont(self):
except SyntaxError:
self.fail("Empty line after a line continuation character is valid.")
Comment thread
pablogsal marked this conversation as resolved.

# See issue-46091
s1 = r"""\
def fib(n):
\
'''Print a Fibonacci series up to n.'''
\
a, b = 0, 1
"""
s2 = r"""\
def fib(n):
'''Print a Fibonacci series up to n.'''
a, b = 0, 1
"""
try:
self.assertEqual(compile(s1, '<string>', 'exec'), compile(s2, '<string>', 'exec'))
except SyntaxError:
self.fail("Indented statement over multiple lines is valid")

def test_continuation_bad_indentation(self):
# Check that code that breaks indentation across multiple lines raises a syntax error

code = r"""\
if x:
y = 1
\
foo = 1
"""

self.assertRaises(IndentationError, exec, code)
Comment thread
pablogsal marked this conversation as resolved.

@support.cpython_only
def test_nested_named_except_blocks(self):
code = ""
Expand Down
100 changes: 99 additions & 1 deletion Lib/test/test_tokenize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
NEWLINE, _generate_tokens_from_c_tokenizer)
from io import BytesIO, StringIO
import unittest
from textwrap import dedent
Comment thread
pablogsal marked this conversation as resolved.
from unittest import TestCase, mock
from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
INVALID_UNDERSCORE_LITERALS)
Expand Down Expand Up @@ -44,7 +45,6 @@ def check_tokenize(self, s, expected):
# The ENDMARKER and final NEWLINE are omitted.
f = BytesIO(s.encode('utf-8'))
result = stringify_tokens_from_source(tokenize(f.readline), s)

self.assertEqual(result,
[" ENCODING 'utf-8' (0, 0) (0, 0)"] +
expected.rstrip().splitlines())
Expand Down Expand Up @@ -2511,7 +2511,105 @@ def get_tokens(string):

self.assertRaises(SyntaxError, get_tokens, "("*1000+"a"+")"*1000)
self.assertRaises(SyntaxError, get_tokens, "]")

def test_continuation_lines_indentation(self):
def get_tokens(string):
return [(kind, string) for (kind, string, *_) in _generate_tokens_from_c_tokenizer(string)]

code = dedent("""
def fib(n):
\\
'''Print a Fibonacci series up to n.'''
\\
a, b = 0, 1
""")

self.check_tokenize(code, """\
NAME 'def' (2, 0) (2, 3)
NAME 'fib' (2, 4) (2, 7)
LPAR '(' (2, 7) (2, 8)
NAME 'n' (2, 8) (2, 9)
RPAR ')' (2, 9) (2, 10)
COLON ':' (2, 10) (2, 11)
NEWLINE '' (2, 11) (2, 11)
INDENT '' (4, -1) (4, -1)
STRING "'''Print a Fibonacci series up to n.'''" (4, 0) (4, 39)
NEWLINE '' (4, 39) (4, 39)
NAME 'a' (6, 0) (6, 1)
COMMA ',' (6, 1) (6, 2)
NAME 'b' (6, 3) (6, 4)
EQUAL '=' (6, 5) (6, 6)
NUMBER '0' (6, 7) (6, 8)
COMMA ',' (6, 8) (6, 9)
NUMBER '1' (6, 10) (6, 11)
NEWLINE '' (6, 11) (6, 11)
DEDENT '' (6, -1) (6, -1)
""")

code_no_cont = dedent("""
def fib(n):
'''Print a Fibonacci series up to n.'''
a, b = 0, 1
""")

self.assertEqual(get_tokens(code), get_tokens(code_no_cont))

code = dedent("""
pass
\\

pass
""")

self.check_tokenize(code, """\
NAME 'pass' (2, 0) (2, 4)
NEWLINE '' (2, 4) (2, 4)
NAME 'pass' (5, 0) (5, 4)
NEWLINE '' (5, 4) (5, 4)
""")

code_no_cont = dedent("""
pass
pass
""")

self.assertEqual(get_tokens(code), get_tokens(code_no_cont))

code = dedent("""
if x:
y = 1
\\
\\
\\
\\
foo = 1
""")

self.check_tokenize(code, """\
NAME 'if' (2, 0) (2, 2)
NAME 'x' (2, 3) (2, 4)
COLON ':' (2, 4) (2, 5)
NEWLINE '' (2, 5) (2, 5)
INDENT '' (3, -1) (3, -1)
NAME 'y' (3, 4) (3, 5)
EQUAL '=' (3, 6) (3, 7)
NUMBER '1' (3, 8) (3, 9)
NEWLINE '' (3, 9) (3, 9)
NAME 'foo' (8, 4) (8, 7)
EQUAL '=' (8, 8) (8, 9)
NUMBER '1' (8, 10) (8, 11)
NEWLINE '' (8, 11) (8, 11)
DEDENT '' (8, -1) (8, -1)
""")

code_no_cont = dedent("""
if x:
y = 1
foo = 1
""")

self.assertEqual(get_tokens(code), get_tokens(code_no_cont))


if __name__ == "__main__":
unittest.main()
2 changes: 2 additions & 0 deletions Misc/NEWS.d/next/Core and Builtins/2021-12-16-00-24-00.bpo-46091.rJ_e_e.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Correctly calculate indentation levels for lines with whitespace character
that are ended by line continuation characters. Patch by Pablo Galindo
46 changes: 33 additions & 13 deletions Parser/tokenizer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1347,6 +1347,24 @@ tok_decimal_tail(struct tok_state *tok)

/* Get next token, after space stripping etc. */

static inline int
tok_continuation_line(struct tok_state *tok) {
int c = tok_nextc(tok);
if (c != '\n') {
tok->done = E_LINECONT;
return -1;
}
c = tok_nextc(tok);
if (c == EOF) {
tok->done = E_EOF;
tok->cur = tok->inp;
return -1;
} else {
tok_backup(tok, c);
}
return c;
}

static int
tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
{
Expand All @@ -1363,6 +1381,7 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
int col = 0;
int altcol = 0;
tok->atbol = 0;
int cont_line_col = 0;
for (;;) {
c = tok_nextc(tok);
if (c == ' ') {
Expand All @@ -1375,14 +1394,23 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
else if (c == '\014') {/* Control-L (formfeed) */
col = altcol = 0; /* For Emacs users */
}
else if (c == '\\') {
// Indentation cannot be split over multiple physical lines
// using backslashes. This means that if we found a backslash
// preceded by whitespace, **the first one we find** determines
// the level of indentation of whatever comes next.
cont_line_col = cont_line_col ? cont_line_col : col;
if ((c = tok_continuation_line(tok)) == -1) {
return ERRORTOKEN;
}
}
else {
break;
}
}
tok_backup(tok, c);
if (c == '#' || c == '\n' || c == '\\') {
if (c == '#' || c == '\n') {
/* Lines with only whitespace and/or comments
and/or a line continuation character
shouldn't affect the indentation and are
not passed to the parser as NEWLINE tokens,
except *totally* empty lines in interactive
Expand All @@ -1403,6 +1431,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
may need to skip to the end of a comment */
}
if (!blankline && tok->level == 0) {
col = cont_line_col ? cont_line_col : col;
altcol = cont_line_col ? cont_line_col : altcol;
if (col == tok->indstack[tok->indent]) {
/* No change */
if (altcol != tok->altindstack[tok->indent]) {
Expand Down Expand Up @@ -1964,19 +1994,9 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)

/* Line continuation */
if (c == '\\') {
c = tok_nextc(tok);
if (c != '\n') {
tok->done = E_LINECONT;
if ((c = tok_continuation_line(tok)) == -1) {
return ERRORTOKEN;
}
c = tok_nextc(tok);
if (c == EOF) {
tok->done = E_EOF;
tok->cur = tok->inp;
return ERRORTOKEN;
} else {
tok_backup(tok, c);
}
tok->cont_line = 1;
goto again; /* Read next line */
}
Expand Down