Skip to content

Commit 3eb554f

Browse files
Issue python#22221: Backported fixes from Python 3 (issue python#18960).
* Now the source encoding declaration on the second line isn't effective if the first line contains anything except a comment. This affects compile(), eval() and exec() too. * IDLE now ignores the source encoding declaration on the second line if the first line contains anything except a comment. * 2to3 and the findnocoding.py script now ignore the source encoding declaration on the second line if the first line contains anything except a comment.
1 parent 423f128 commit 3eb554f

6 files changed

Lines changed: 52 additions & 5 deletions

File tree

‎Lib/idlelib/IOBinding.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
encoding = encoding.lower()
7373

7474
coding_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
75+
blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)')
7576

7677
class EncodingMessage(SimpleDialog):
7778
"Inform user that an encoding declaration is needed."
@@ -130,6 +131,8 @@ def coding_spec(str):
130131
match = coding_re.match(line)
131132
if match is not None:
132133
break
134+
if not blank_re.match(line):
135+
return None
133136
else:
134137
return None
135138
name = match.group(1)

‎Lib/lib2to3/pgen2/tokenize.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@ def compat(self, token, iterable):
237237
toks_append(tokval)
238238

239239
cookie_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
240+
blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)')
240241

241242
def _get_normal_name(orig_enc):
242243
"""Imitates get_normal_name in tokenizer.c."""
@@ -309,6 +310,8 @@ def find_cookie(line):
309310
encoding = find_cookie(first)
310311
if encoding:
311312
return encoding, [first]
313+
if not blank_re.match(first):
314+
return default, [first]
312315

313316
second = read_or_stop()
314317
if not second:

‎Lib/test/test_compile.py‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -412,9 +412,24 @@ def test_lambda_doc(self):
412412
l = lambda: "foo"
413413
self.assertIsNone(l.__doc__)
414414

415-
def test_unicode_encoding(self):
415+
@test_support.requires_unicode
416+
def test_encoding(self):
417+
code = b'# -*- coding: badencoding -*-\npass\n'
418+
self.assertRaises(SyntaxError, compile, code, 'tmp', 'exec')
416419
code = u"# -*- coding: utf-8 -*-\npass\n"
417420
self.assertRaises(SyntaxError, compile, code, "tmp", "exec")
421+
code = 'u"\xc2\xa4"\n'
422+
self.assertEqual(eval(code), u'\xc2\xa4')
423+
code = u'u"\xc2\xa4"\n'
424+
self.assertEqual(eval(code), u'\xc2\xa4')
425+
code = '# -*- coding: latin1 -*-\nu"\xc2\xa4"\n'
426+
self.assertEqual(eval(code), u'\xc2\xa4')
427+
code = '# -*- coding: utf-8 -*-\nu"\xc2\xa4"\n'
428+
self.assertEqual(eval(code), u'\xa4')
429+
code = '# -*- coding: iso8859-15 -*-\nu"\xc2\xa4"\n'
430+
self.assertEqual(eval(code), test_support.u(r'\xc2\u20ac'))
431+
code = 'u"""\\\n# -*- coding: utf-8 -*-\n\xc2\xa4"""\n'
432+
self.assertEqual(eval(code), u'# -*- coding: utf-8 -*-\n\xc2\xa4')
418433

419434
def test_subscripts(self):
420435
# SF bug 1448804

‎Misc/NEWS‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ What's New in Python 2.7.9?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #22221: Now the source encoding declaration on the second line isn't
14+
effective if the first line contains anything except a comment.
15+
1316
- Issue #22023: Fix ``%S``, ``%R`` and ``%V`` formats of
1417
:c:func:`PyUnicode_FromFormat`.
1518

@@ -124,6 +127,9 @@ _ Issue #21597: The separator between the turtledemo text pane and the drawing
124127
IDLE
125128
----
126129

130+
- Issue #22221: IDLE now ignores the source encoding declaration on the second
131+
line if the first line contains anything except a comment.
132+
127133
- Issue #17390: Adjust Editor window title; remove 'Python',
128134
move version to end.
129135

@@ -140,6 +146,10 @@ Extension Modules
140146
Tools/Demos
141147
-----------
142148

149+
- Issue #22221: 2to3 and the findnocoding.py script now ignore the source
150+
encoding declaration on the second line if the first line contains anything
151+
except a comment.
152+
143153
- Issue #22201: Command-line interface of the zipfile module now correctly
144154
extracts ZIP files with directory entries. Patch by Ryan Wilson.
145155

‎Parser/tokenizer.c‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,11 +259,25 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
259259
char * cs;
260260
int r = 1;
261261

262-
if (tok->cont_line)
262+
if (tok->cont_line) {
263263
/* It's a continuation line, so it can't be a coding spec. */
264+
tok->read_coding_spec = 1;
264265
return 1;
266+
}
265267
cs = get_coding_spec(line, size);
266-
if (cs != NULL) {
268+
if (!cs) {
269+
Py_ssize_t i;
270+
for (i = 0; i < size; i++) {
271+
if (line[i] == '#' || line[i] == '\n' || line[i] == '\r')
272+
break;
273+
if (line[i] != ' ' && line[i] != '\t' && line[i] != '\014') {
274+
/* Stop checking coding spec after a line containing
275+
* anything except a comment. */
276+
tok->read_coding_spec = 1;
277+
break;
278+
}
279+
}
280+
} else {
267281
tok->read_coding_spec = 1;
268282
if (tok->encoding == NULL) {
269283
assert(tok->decoding_state == 1); /* raw */
@@ -688,7 +702,7 @@ decode_str(const char *input, int single, struct tok_state *tok)
688702
if (newl[0]) {
689703
if (!check_coding_spec(str, newl[0] - str, tok, buf_setreadl))
690704
return error_ret(tok);
691-
if (tok->enc == NULL && newl[1]) {
705+
if (tok->enc == NULL && !tok->read_coding_spec && newl[1]) {
692706
if (!check_coding_spec(newl[0]+1, newl[1] - newl[0],
693707
tok, buf_setreadl))
694708
return error_ret(tok);

‎Tools/scripts/findnocoding.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ def walk_python_files(self, paths, *args, **kwargs):
3333

3434

3535
decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
36+
blank_re = re.compile(r'^[ \t\f]*(?:[#\r\n]|$)')
3637

3738
def get_declaration(line):
3839
match = decl_re.match(line)
@@ -57,7 +58,8 @@ def needs_declaration(fullpath):
5758
line1 = infile.readline()
5859
line2 = infile.readline()
5960

60-
if get_declaration(line1) or get_declaration(line2):
61+
if (get_declaration(line1) or
62+
blank_re.match(line1) and get_declaration(line2)):
6163
# the file does have an encoding declaration, so trust it
6264
infile.close()
6365
return False

0 commit comments

Comments
 (0)