Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions Lib/test/test_tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -944,6 +944,43 @@ async def bar(): pass
DEDENT '' (7, 0) (7, 0)
""")

def test_linecont(self):
self.check_tokenize("x = \\\n1", """\
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (2, 0) (2, 1)
""")
self.check_tokenize("""x = 1
\\

y = 1""", """\
NAME 'x' (1, 0) (1, 1)
OP '=' (1, 2) (1, 3)
NUMBER '1' (1, 4) (1, 5)
NEWLINE '\\n' (1, 5) (1, 6)
NL '\\n' (2, 1) (2, 2)
NL '\\n' (3, 0) (3, 1)
NAME 'y' (4, 0) (4, 1)
OP '=' (4, 2) (4, 3)
NUMBER '1' (4, 4) (4, 5)
""")
self.check_tokenize("""def f(x):
\\
return x""", """\
NAME 'def' (1, 0) (1, 3)
NAME 'f' (1, 4) (1, 5)
OP '(' (1, 5) (1, 6)
NAME 'x' (1, 6) (1, 7)
OP ')' (1, 7) (1, 8)
OP ':' (1, 8) (1, 9)
NEWLINE '\\n' (1, 9) (1, 10)
NL '\\n' (2, 5) (2, 6)
INDENT ' ' (3, 0) (3, 2)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Before PR the indent would be 4 spaces, not 2.

NAME 'return' (3, 2) (3, 8)
NAME 'x' (3, 9) (3, 10)
DEDENT '' (4, 0) (4, 0)
""")

class GenerateTokensTest(TokenizeTest):
def check_tokenize(self, s, expected):
# Format the tokens in s in a table format.
Expand Down Expand Up @@ -1593,6 +1630,10 @@ def test_backslash_continuation(self):
"# This also\n")
self.check_roundtrip("# Comment \\\n"
"x = 0")
self.check_roundtrip("x = 1\n"
"\\\n"
"\n"
"y = 1")

def test_string_concatenation(self):
# Two string literals on the same line
Expand Down
5 changes: 5 additions & 0 deletions Lib/tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,11 @@ def _tokenize(readline, encoding):
yield TokenInfo(NL, line[pos:],
(lnum, pos), (lnum, len(line)), line)
continue
elif line[pos] == '\\':
if pos + 1 != max and line[pos + 1] in '\r\n':
yield TokenInfo(NL, line[pos + 1:],
(lnum, pos + 1), (lnum, len(line)), line)
continue

if column > indents[-1]: # count indents or dedents
indents.append(column)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Tokenizer no longer emits NEWLINE token on empty continuation lines.
23 changes: 20 additions & 3 deletions Parser/tokenizer.c
Original file line number Diff line number Diff line change
Expand Up @@ -1136,7 +1136,6 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
break;
}
}
tok_backup(tok, c);
if (c == '#' || c == '\n') {
/* Lines with only whitespace and/or comments
shouldn't affect the indentation and are
Expand All @@ -1149,9 +1148,22 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
else {
blankline = 1; /* Ignore completely */
}
tok_backup(tok, c);
/* We can't jump back right here since we still
may need to skip to the end of a comment */
}
else if (c == '\\') {
c = tok_nextc(tok);
if (c == '\n') {
/* Line continuation of a blank line */
blankline = 1;
}
tok_backup(tok, c);
tok_backup(tok, '\\');
}
else {
tok_backup(tok, c);
}
if (!blankline && tok->level == 0) {
if (col == tok->indstack[tok->indent]) {
/* No change */
Expand Down Expand Up @@ -1688,8 +1700,13 @@ tok_get(struct tok_state *tok, char **p_start, char **p_end)
} else {
tok_backup(tok, c);
}
tok->cont_line = 1;
goto again; /* Read next line */
if (blankline && tok->level == 0) {
tok->atbol = 1;
}
else {
tok->cont_line = 1;
}
goto nextline; /* Read next line */
}

/* Check for two-character token */
Expand Down