Skip to content

bpo-29812: Improve testing of token and tokenize #681

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ammaraskar wants to merge 4 commits into from
Closed

bpo-29812: Improve testing of token and tokenize #681

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bd26376
bpo-29812: Improve testing of token and tokenize
ammaraskar Mar 16, 2017
635f0e7
Changes for berker's review
ammaraskar Apr 11, 2017
3d4a043
Preserve newline conventions for skeleton file
ammaraskar Jun 24, 2017
7521597
Add newly added non literal tokens to test
ammaraskar Jun 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions Lib/test/test_token.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import filecmp
import os
import subprocess
import sys
from test import support

@berkerpeksag berkerpeksag Jul 3, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please split stdlib and non-stdlib modules.

import token
import unittest


TOKEN_FILE = support.findfile('token.py')
TOKEN_INCLUDE_FILE = os.path.join(os.path.split(__file__)[0],
'..', '..', 'Include', 'token.h')
TEST_PY_FILE = 'token_test.py'


class TestTokenGeneration(unittest.TestCase):

def _copy_file_without_generated_tokens(self, source_file, dest_file):
with open(source_file, 'rb') as fp:
lines = fp.readlines()
nl = lines[0][len(lines[0].strip()):]
with open(dest_file, 'wb') as fp:
fp.writelines(lines[:lines.index(b"#--start constants--" + nl) + 1])
fp.writelines(lines[lines.index(b"#--end constants--" + nl):])
self.addCleanup(support.unlink, dest_file)

def _generate_tokens(self, skeleton_file, target_token_py_file):
proc = subprocess.Popen([sys.executable,

@berkerpeksag berkerpeksag Jul 3, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we use the new subprocess.run() interface?

TOKEN_FILE,
skeleton_file,
target_token_py_file], stderr=subprocess.PIPE)
stderr = proc.communicate()[1]
return proc.returncode, stderr

@unittest.skipUnless(os.path.exists(TOKEN_FILE),
'test only works from source build directory')
def test_real_token_file(self):
self._copy_file_without_generated_tokens(TOKEN_FILE, TEST_PY_FILE)
self.assertFalse(filecmp.cmp(TOKEN_FILE, TEST_PY_FILE))
self.assertEqual((0, b''), self._generate_tokens(TOKEN_INCLUDE_FILE,
TEST_PY_FILE))
self.assertTrue(filecmp.cmp(TOKEN_FILE, TEST_PY_FILE))

def test_missing_output_file_causes_error(self):
rc, stderr = self._generate_tokens(os.devnull, 'not_here.txt')
self.assertNotEqual(rc, 0)
self.assertIn(b'I/O error', stderr)
self.assertIn(b'not_here.txt', stderr)


if __name__ == '__main__':
unittest.main()
24 changes: 23 additions & 1 deletion Lib/test/test_tokenize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from test import support
from tokenize import (tokenize, _tokenize, untokenize, NUMBER, NAME, OP,
STRING, ENDMARKER, ENCODING, tok_name, detect_encoding,
open as tokenize_open, Untokenizer)
open as tokenize_open, Untokenizer, EXACT_TOKEN_TYPES)
from io import BytesIO
from unittest import TestCase, mock
from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
Expand Down Expand Up @@ -1351,6 +1351,28 @@ def assertExactTypeEqual(self, opstr, *optypes):
self.assertEqual(tok_name[tokens[1 + num_optypes].exact_type],
tok_name[token.ENDMARKER])

def test_all_literal_tokens(self):
non_literals = {
token.ENDMARKER, token.NAME, token.NUMBER,
token.STRING, token.NEWLINE, token.INDENT,
token.DEDENT, token.OP, token.ERRORTOKEN,
token.AWAIT, token.ASYNC,
token.COMMENT, token.NL, token.ENCODING,
token.N_TOKENS, token.NT_OFFSET,
}

for tok in range(token.N_TOKENS):
if tok in non_literals:
continue
name = tok_name[tok]
with self.subTest(name=name):
self.assertIn(tok, EXACT_TOKEN_TYPES.values())

@berkerpeksag berkerpeksag Jul 3, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking at CI logs of an earlier commit, I wonder if we should make the following message clearer:

======================================================================
FAIL: test_all_literal_tokens (test.test_tokenize.TestTokenize) (name='NL')
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/travis/build/python/cpython/Lib/test/test_tokenize.py", line 1368, in test_all_literal_tokens
    self.assertIn(tok, EXACT_TOKEN_TYPES.values())
AssertionError: 58 not found in dict_values([7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 52, 51, 49, 50])


for tok in non_literals:
name = tok_name[tok]
with self.subTest(name=name):
self.assertNotIn(tok, EXACT_TOKEN_TYPES.values())

def test_exact_type(self):
self.assertExactTypeEqual('()', token.LPAR, token.RPAR)
self.assertExactTypeEqual('[]', token.LSQB, token.RSQB)
Expand Down
22 changes: 13 additions & 9 deletions Lib/token.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,33 +129,37 @@ def _main():
comment = comment_match.group(1)
tokens[prev_val]['comment'] = comment
keys = sorted(tokens.keys())
# load the output skeleton from the target:
# load the output skeleton from the target, taking
# care to preserve its newline convention.
# newline detection code from keyword.py
try:
fp = open(outFileName)
fp = open(outFileName, newline='')
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(2)
with fp:
format = fp.read().split("\n")
format = fp.readlines()
nl = format[0][len(format[0].strip()):] if format else '\n'

@berkerpeksag berkerpeksag Jul 3, 2017

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd use more descriptive name like newline.


try:
start = format.index("#--start constants--") + 1
end = format.index("#--end constants--")
start = format.index("#--start constants--" + nl) + 1
end = format.index("#--end constants--" + nl)
except ValueError:
sys.stderr.write("target does not contain format markers")
sys.exit(3)
lines = []
for key in keys:
lines.append("%s = %d" % (tokens[key]["token"], key))
lines.append("%s = %d%s" % (tokens[key]["token"], key, nl))
if "comment" in tokens[key]:
lines.append("# %s" % tokens[key]["comment"])
lines.append("# %s%s" % (tokens[key]["comment"], nl))
format[start:end] = lines
try:
fp = open(outFileName, 'w')
fp = open(outFileName, 'w', newline='')
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(4)
with fp:
fp.write("\n".join(format))
fp.writelines(format)


if __name__ == "__main__":
Expand Down