Skip to content

[WIP] bpo-30638: Add regen-token and regen-symbol make targets #5679

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
zware wants to merge 4 commits into from
Closed

[WIP] bpo-30638: Add regen-token and regen-symbol make targets #5679

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e785937
bpo-30638: Add regen-token make target
zware Feb 14, 2018
2e8e9e4
Regen token.py, fix tokenize.py accordingly
zware Feb 14, 2018
c4e741e
Add regen-symbol target
zware Feb 14, 2018
748cb80
Try to specify a newer Python on Travis
zware Feb 14, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ language: c
dist: trusty
sudo: false
group: beta
python: 3.6

# To cache doc-building dependencies and C compiler output.
cache:
Expand Down Expand Up @@ -42,7 +43,6 @@ matrix:
- os: linux
language: python
# Build the docs against a stable version of Python so code bugs don't hold up doc-related PRs.
python: 3.6
env: TESTING=docs
before_script:
- cd Doc
Expand Down
17 changes: 9 additions & 8 deletions Lib/token.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,12 @@
ELLIPSIS = 52
# Don't forget to update the table _PyParser_TokenNames in tokenizer.c!
OP = 53
AWAIT = 54
ASYNC = 55
ERRORTOKEN = 56
ERRORTOKEN = 54
# These aren't used by the C tokenizer but are needed for tokenize.py
COMMENT = 57
NL = 58
ENCODING = 59
N_TOKENS = 60
COMMENT = 55
NL = 56
ENCODING = 57
N_TOKENS = 58
# Special definitions for cooperation with parser
NT_OFFSET = 256
#--end constants--
Expand Down Expand Up @@ -100,6 +98,9 @@ def _main():
outFileName = "Lib/token.py"
if len(args) > 1:
outFileName = args[1]
skeletonFileName = __file__
if len(args) > 2:
skeletonFileName = args[2]
try:
fp = open(inFileName)
except OSError as err:
Expand Down Expand Up @@ -131,7 +132,7 @@ def _main():
keys = sorted(tokens.keys())
# load the output skeleton from the target:
try:
fp = open(outFileName)
fp = open(skeletonFileName)
except OSError as err:
sys.stderr.write("I/O error: %s\n" % str(err))
sys.exit(2)
Expand Down
2 changes: 1 addition & 1 deletion Lib/tokenize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,7 +284,7 @@ def compat(self, token, iterable):
self.encoding = tokval
continue

if toknum in (NAME, NUMBER, ASYNC, AWAIT):
if toknum in (NAME, NUMBER):
tokval += ' '

# Insert a space between two consecutive strings
Expand Down
12 changes: 11 additions & 1 deletion Makefile.pre.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -733,7 +733,8 @@ regen-importlib: Programs/_freeze_importlib
############################################################################
# Regenerate all generated files

regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-ast regen-importlib
regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar \
regen-ast regen-importlib regen-token regen-symbol

############################################################################
# Special rules for object files
Expand Down Expand Up @@ -817,6 +818,15 @@ Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c

Parser/pgenmain.o: $(srcdir)/Include/parsetok.h

.PHONY=regen-token regen-symbol
regen-token:
$(PYTHON_FOR_REGEN) $(srcdir)/Lib/token.py $(srcdir)/Include/token.h $(srcdir)/Lib/token.py.new
$(UPDATE_FILE) $(srcdir)/Lib/token.py $(srcdir)/Lib/token.py.new

regen-symbol:
$(PYTHON_FOR_REGEN) $(srcdir)/Lib/symbol.py $(srcdir)/Include/graminit.h $(srcdir)/Lib/symbol.py.new $(srcdir)/Lib/symbol.py
$(UPDATE_FILE) $(srcdir)/Lib/symbol.py $(srcdir)/Lib/symbol.py.new

.PHONY=regen-ast
regen-ast:
# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
Expand Down