Skip to content

bpo-40334: Add support for feature_version in new PEG parser #19827

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gvanrossum merged 2 commits into from
May 1, 2020
Merged

bpo-40334: Add support for feature_version in new PEG parser #19827

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
966f7e9
bpo-40334: Add support for feature_version in new PEG parser
lysnikolaou May 1, 2020
07e5a41
Corectly handle NULL flags
lysnikolaou May 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 51 additions & 29 deletions Grammar/python.gram
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,30 +80,34 @@ compound_stmt[stmt_ty]:
# NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield'
assignment:
| a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] {
_Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) }
CHECK_VERSION(
6,
"Variable annotation syntax is",
_Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA)
) }
| a=('(' b=inside_paren_ann_assign_target ')' { b }
| ann_assign_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] {
_Py_AnnAssign(a, b, c, 0, EXTRA)}
CHECK_VERSION(6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) }
| a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) tc=[TYPE_COMMENT] {
_Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| a=target b=augassign c=(yield_expr | star_expressions) {
_Py_AugAssign(a, b->kind, c, EXTRA) }
| invalid_assignment

augassign[AugOperator*]:
| '+=' {_PyPegen_augoperator(p, Add)}
| '-=' {_PyPegen_augoperator(p, Sub)}
| '*=' {_PyPegen_augoperator(p, Mult)}
| '@=' {_PyPegen_augoperator(p, MatMult)}
| '/=' {_PyPegen_augoperator(p, Div)}
| '%=' {_PyPegen_augoperator(p, Mod)}
| '&=' {_PyPegen_augoperator(p, BitAnd)}
| '|=' {_PyPegen_augoperator(p, BitOr)}
| '^=' {_PyPegen_augoperator(p, BitXor)}
| '<<=' {_PyPegen_augoperator(p, LShift)}
| '>>=' {_PyPegen_augoperator(p, RShift)}
| '**=' {_PyPegen_augoperator(p, Pow)}
| '//=' {_PyPegen_augoperator(p, FloorDiv)}
| '+=' { _PyPegen_augoperator(p, Add) }
| '-=' { _PyPegen_augoperator(p, Sub) }
| '*=' { _PyPegen_augoperator(p, Mult) }
| '@=' { CHECK_VERSION(5, "The '@' operator is", _PyPegen_augoperator(p, MatMult)) }
| '/=' { _PyPegen_augoperator(p, Div) }
| '%=' { _PyPegen_augoperator(p, Mod) }
| '&=' { _PyPegen_augoperator(p, BitAnd) }
| '|=' { _PyPegen_augoperator(p, BitOr) }
| '^=' { _PyPegen_augoperator(p, BitXor) }
| '<<=' { _PyPegen_augoperator(p, LShift) }
| '>>=' { _PyPegen_augoperator(p, RShift) }
| '**=' { _PyPegen_augoperator(p, Pow) }
| '//=' { _PyPegen_augoperator(p, FloorDiv) }

global_stmt[stmt_ty]: 'global' a=','.NAME+ {
_Py_Global(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) }
Expand Down Expand Up @@ -156,14 +160,20 @@ while_stmt[stmt_ty]:
| 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) }

for_stmt[stmt_ty]:
| is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
(is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
_Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| ASYNC 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] {
CHECK_VERSION(5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
Comment on lines +163 to +166

@gvanrossum gvanrossum May 1, 2020

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Heh. After all my efforts to combine the two rules... :-) (Seriously, it's probably clearer this way anyways.)

@lysnikolaou lysnikolaou May 1, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think so, too.


with_stmt[stmt_ty]:
| is_async=[ASYNC] 'with' '(' a=','.with_item+ ')' ':' b=block {
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) }
| is_async=[ASYNC] 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
(is_async ? _Py_AsyncWith : _Py_With)(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| 'with' '(' a=','.with_item+ ')' ':' b=block {
_Py_With(a, b, NULL, EXTRA) }
| 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
_Py_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| ASYNC 'with' '(' a=','.with_item+ ')' ':' b=block {
CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NULL, EXTRA)) }
| ASYNC 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block {
CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) }
with_item[withitem_ty]:
| e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) }

Expand All @@ -188,10 +198,18 @@ function_def[stmt_ty]:
| function_def_raw

function_def_raw[stmt_ty]:
| is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
(is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id,
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
_Py_FunctionDef(n->v.Name.id,
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) }
| ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block {
CHECK_VERSION(
5,
"Async functions are",
_Py_AsyncFunctionDef(n->v.Name.id,
(params) ? params : CHECK(_PyPegen_empty_arguments(p)),
b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA)
) }
func_type_comment[PyObject*]:
| NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t } # Must be followed by indented block
| invalid_double_type_comments
Expand Down Expand Up @@ -399,7 +417,7 @@ term[expr_ty]:
| a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) }
| a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) }
| a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) }
| a=term '@' b=factor { _Py_BinOp(a, MatMult, b, EXTRA) }
| a=term '@' b=factor { CHECK_VERSION(5, "The '@' operator is", _Py_BinOp(a, MatMult, b, EXTRA)) }
| factor
factor[expr_ty] (memo):
| '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) }
Expand All @@ -410,7 +428,7 @@ power[expr_ty]:
| a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) }
| await_primary
await_primary[expr_ty] (memo):
| AWAIT a=primary { _Py_Await(a, EXTRA) }
| AWAIT a=primary { CHECK_VERSION(5, "Await expressions are", _Py_Await(a, EXTRA)) }
| primary
primary[expr_ty]:
| a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) }
Expand Down Expand Up @@ -469,8 +487,12 @@ kvpair[KeyValuePair*]:
| '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) }
| a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) }
for_if_clauses[asdl_seq*]:
| a=(y=[ASYNC] 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })*
{ _Py_comprehension(a, b, c, y != NULL, p->arena) })+ { a }
| for_if_clause+
for_if_clause[comprehension_ty]:
| ASYNC 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* {
CHECK_VERSION(6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) }
| 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* {
_Py_comprehension(a, b, c, 0, p->arena) }

yield_expr[expr_ty]:
| 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) }
Expand Down
6 changes: 0 additions & 6 deletions Lib/test/test_type_comments.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,6 @@ def test_funcdef(self):
self.assertEqual(tree.body[0].type_comment, None)
self.assertEqual(tree.body[1].type_comment, None)

@support.skip_if_new_parser("Pegen does not support feature_version yet")
def test_asyncdef(self):
for tree in self.parse_all(asyncdef, minver=5):
self.assertEqual(tree.body[0].type_comment, "() -> int")
Expand All @@ -261,27 +260,22 @@ def test_asyncdef(self):
self.assertEqual(tree.body[0].type_comment, None)
self.assertEqual(tree.body[1].type_comment, None)

@support.skip_if_new_parser("Pegen does not support feature_version yet")
def test_asyncvar(self):
for tree in self.parse_all(asyncvar, maxver=6):
pass

@support.skip_if_new_parser("Pegen does not support feature_version yet")
def test_asynccomp(self):
for tree in self.parse_all(asynccomp, minver=6):
pass

@support.skip_if_new_parser("Pegen does not support feature_version yet")
def test_matmul(self):
for tree in self.parse_all(matmul, minver=5):
pass

@support.skip_if_new_parser("Pegen does not support feature_version yet")
def test_fstring(self):
for tree in self.parse_all(fstring, minver=6):
pass

@support.skip_if_new_parser("Pegen does not support feature_version yet")
def test_underscorednumber(self):
for tree in self.parse_all(underscorednumber, minver=6):
pass
Expand Down
Loading