python · Rosuav · Apr 1, 2018 · Apr 2, 2018 · Apr 3, 2018 · Apr 8, 2018
diff --git a/Grammar/Grammar b/Grammar/Grammar
@@ -83,7 +83,7 @@ with_item: test ['as' expr]
 except_clause: 'except' [test ['as' NAME]]
 suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
 
-test: or_test ['if' or_test 'else' test] | lambdef
+test: or_test ['if' or_test 'else' test] | lambdef | or_test ':=' test
 test_nocond: or_test | lambdef_nocond
 lambdef: 'lambda' [varargslist] ':' test
 lambdef_nocond: 'lambda' [varargslist] ':' test_nocond

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
@@ -211,12 +211,13 @@ struct _stmt {
 enum _expr_kind {BoolOp_kind=1, BinOp_kind=2, UnaryOp_kind=3, Lambda_kind=4,
                   IfExp_kind=5, Dict_kind=6, Set_kind=7, ListComp_kind=8,
                   SetComp_kind=9, DictComp_kind=10, GeneratorExp_kind=11,
-                  Await_kind=12, Yield_kind=13, YieldFrom_kind=14,
-                  Compare_kind=15, Call_kind=16, Num_kind=17, Str_kind=18,
-                  FormattedValue_kind=19, JoinedStr_kind=20, Bytes_kind=21,
-                  NameConstant_kind=22, Ellipsis_kind=23, Constant_kind=24,
-                  Attribute_kind=25, Subscript_kind=26, Starred_kind=27,
-                  Name_kind=28, List_kind=29, Tuple_kind=30};
+                  AssignExp_kind=12, Await_kind=13, Yield_kind=14,
+                  YieldFrom_kind=15, Compare_kind=16, Call_kind=17,
+                  Num_kind=18, Str_kind=19, FormattedValue_kind=20,
+                  JoinedStr_kind=21, Bytes_kind=22, NameConstant_kind=23,
+                  Ellipsis_kind=24, Constant_kind=25, Attribute_kind=26,
+                  Subscript_kind=27, Starred_kind=28, Name_kind=29,
+                  List_kind=30, Tuple_kind=31};
 struct _expr {
     enum _expr_kind kind;
     union {
@@ -277,6 +278,11 @@ struct _expr {
             asdl_seq *generators;
         } GeneratorExp;
 
+        struct {
+            expr_ty target;
+            expr_ty value;
+        } AssignExp;
+
         struct {
             expr_ty value;
         } Await;
@@ -558,6 +564,9 @@ expr_ty _Py_DictComp(expr_ty key, expr_ty value, asdl_seq * generators, int
 #define GeneratorExp(a0, a1, a2, a3, a4) _Py_GeneratorExp(a0, a1, a2, a3, a4)
 expr_ty _Py_GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, int
                          col_offset, PyArena *arena);
+#define AssignExp(a0, a1, a2, a3, a4) _Py_AssignExp(a0, a1, a2, a3, a4)
+expr_ty _Py_AssignExp(expr_ty target, expr_ty value, int lineno, int
+                      col_offset, PyArena *arena);
 #define Await(a0, a1, a2, a3) _Py_Await(a0, a1, a2, a3)
 expr_ty _Py_Await(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Yield(a0, a1, a2, a3) _Py_Yield(a0, a1, a2, a3)

diff --git a/Include/symtable.h b/Include/symtable.h
@@ -42,6 +42,7 @@ typedef struct _symtable_entry {
     PyObject *ste_varnames;  /* list of function parameters */
     PyObject *ste_children;  /* list of child blocks */
     PyObject *ste_directives;/* locations of global and nonlocal statements */
+    PyObject *ste_classsyms; /* ste_symbols for the surrounding class of a comprehension */
     _Py_block_ty ste_type;   /* module, class, or function */
     int ste_nested;      /* true if block is nested */
     unsigned ste_free : 1;        /* true if block has free variables */
@@ -92,6 +93,7 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define DEF_FREE_CLASS 2<<5    /* free variable from class's method */
 #define DEF_IMPORT 2<<6        /* assignment occurred via import */
 #define DEF_ANNOT 2<<7         /* this name is annotated */
+#define DEF_EARLYBIND 2<<8     /* name requires early binding semantics (via function args) */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 

diff --git a/Include/token.h b/Include/token.h
@@ -62,14 +62,15 @@ extern "C" {
 #define ATEQUAL         50
 #define RARROW          51
 #define ELLIPSIS        52
+#define COLONEQUAL      53
 /* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */
-#define OP              53
-#define ERRORTOKEN      54
+#define OP              54
+#define ERRORTOKEN      55
 /* These aren't used by the C tokenizer but are needed for tokenize.py */
-#define COMMENT         55
-#define NL              56
-#define ENCODING                57
-#define N_TOKENS        58
+#define COMMENT         56
+#define NL              57
+#define ENCODING                58
+#define N_TOKENS        59
 
 /* Special definitions for cooperation with parser */
 

diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py
@@ -146,12 +146,12 @@ def bug1333982(x=[]):
 %3d           0 LOAD_CONST               1 (0)
               2 POP_JUMP_IF_TRUE        26
               4 LOAD_GLOBAL              0 (AssertionError)
-              6 LOAD_CONST               2 (<code object <listcomp> at 0x..., file "%s", line %d>)
-              8 LOAD_CONST               3 ('bug1333982.<locals>.<listcomp>')
-             10 MAKE_FUNCTION            0
-             12 LOAD_FAST                0 (x)
-             14 GET_ITER
-             16 CALL_FUNCTION            1
+              6 LOAD_CLOSURE             0 (x)
+              8 BUILD_TUPLE              1
+             10 LOAD_CONST               2 (<code object <listcomp> at 0x..., file "%s", line %d>)
+             12 LOAD_CONST               3 ('bug1333982.<locals>.<listcomp>')
+             14 MAKE_FUNCTION            8 (closure)
+             16 CALL_FUNCTION            0
 
 %3d          18 LOAD_CONST               4 (1)
              20 BINARY_ADD
@@ -362,14 +362,13 @@ def foo(x):
 dis_nested_1 = """%s
 Disassembly of <code object foo at 0x..., file "%s", line %d>:
 %3d           0 LOAD_CLOSURE             0 (x)
-              2 BUILD_TUPLE              1
-              4 LOAD_CONST               1 (<code object <listcomp> at 0x..., file "%s", line %d>)
-              6 LOAD_CONST               2 ('_h.<locals>.foo.<locals>.<listcomp>')
-              8 MAKE_FUNCTION            8 (closure)
-             10 LOAD_DEREF               1 (y)
-             12 GET_ITER
-             14 CALL_FUNCTION            1
-             16 RETURN_VALUE
+              2 LOAD_CLOSURE             1 (y)
+              4 BUILD_TUPLE              2
+              6 LOAD_CONST               1 (<code object <listcomp> at 0x..., file "%s", line %d>)
+              8 LOAD_CONST               2 ('_h.<locals>.foo.<locals>.<listcomp>')
+             10 MAKE_FUNCTION            8 (closure)
+             12 CALL_FUNCTION            0
+             14 RETURN_VALUE
 """ % (dis_nested_0,
        __file__,
        _h.__code__.co_firstlineno + 1,
@@ -381,15 +380,16 @@ def foo(x):
 dis_nested_2 = """%s
 Disassembly of <code object <listcomp> at 0x..., file "%s", line %d>:
 %3d           0 BUILD_LIST               0
-              2 LOAD_FAST                0 (.0)
-        >>    4 FOR_ITER                12 (to 18)
-              6 STORE_FAST               1 (z)
-              8 LOAD_DEREF               0 (x)
-             10 LOAD_FAST                1 (z)
-             12 BINARY_ADD
-             14 LIST_APPEND              2
-             16 JUMP_ABSOLUTE            4
-        >>   18 RETURN_VALUE
+              2 LOAD_DEREF               1 (y)
+              4 GET_ITER
+        >>    6 FOR_ITER                12 (to 20)
+              8 STORE_FAST               0 (z)
+             10 LOAD_DEREF               0 (x)
+             12 LOAD_FAST                0 (z)
+             14 BINARY_ADD
+             16 LIST_APPEND              2
+             18 JUMP_ABSOLUTE            6
+        >>   20 RETURN_VALUE
 """ % (dis_nested_1,
        __file__,
        _h.__code__.co_firstlineno + 3,

diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py
@@ -1814,13 +1814,6 @@ def printsolution(self, x):
 [None]
 
 
-Yield is allowed only in the outermost iterable in generator expression:
-
->>> def f(): list(i for i in [(yield 26)])
->>> type(f())
-<class 'generator'>
-
-
 A yield expression with augmented assignment.
 
 >>> def coroutine(seq):
@@ -1846,6 +1839,11 @@ def printsolution(self, x):
 
 Check some syntax errors for yield expressions:
 
+>>> def f(): list(i for i in [(yield 26)])
+Traceback (most recent call last):
+  ...
+SyntaxError: 'yield' inside generator expression
+
 >>> f=lambda: (yield 1),(yield 2)
 Traceback (most recent call last):
   ...

diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py
@@ -972,10 +972,12 @@ def g(): f((yield from ()), 1)
 
     def test_yield_in_comprehensions(self):
         # Check yield in comprehensions
-        def g(): [x for x in [(yield 1)]]
-        def g(): [x for x in [(yield from ())]]
 
         check = self.check_syntax_error
+        check("def g(): [x for x in [(yield 1)]]",
+              "'yield' inside list comprehension")
+        check("def g(): [x for x in [(yield from ())]]",
+              "'yield' inside list comprehension")
         check("def g(): [(yield x) for x in ()]",
               "'yield' inside list comprehension")
         check("def g(): [x for x in () if not (yield x)]",
@@ -1409,14 +1411,9 @@ def test_genexps(self):
         check_syntax_error(self, "foo(100, x for x in range(10))")
 
     def test_comprehension_specials(self):
-        # test for outmost iterable precomputation
+        # test that the outmost iterable is not precomputed
         x = 10; g = (i for i in range(x)); x = 5
-        self.assertEqual(len(list(g)), 10)
-
-        # This should hold, since we're only precomputing outmost iterable.
-        x = 10; t = False; g = ((i,j) for i in range(x) if t for j in range(x))
-        x = 5; t = True;
-        self.assertEqual([(i,j) for i in range(10) for j in range(5)], list(g))
+        self.assertEqual(len(list(g)), 5)
 
         # Grammar allows multiple adjacent 'if's in listcomps and genexps,
         # even though it's silly. Make sure it works (ifelse broke this.)

diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py
@@ -3403,17 +3403,6 @@ def test(*args, **kwargs):
         ba = sig.bind(args=1)
         self.assertEqual(ba.arguments, {'kwargs': {'args': 1}})
 
-    @cpython_only
-    def test_signature_bind_implicit_arg(self):
-        # Issue #19611: getcallargs should work with set comprehensions
-        def make_set():
-            return {z * z for z in range(5)}
-        setcomp_code = make_set.__code__.co_consts[1]
-        setcomp_func = types.FunctionType(setcomp_code, {})
-
-        iterator = iter(range(5))
-        self.assertEqual(self.call(setcomp_func, iterator), {0, 1, 4, 9, 16})
-
 
 class TestBoundArguments(unittest.TestCase):
     def test_signature_bound_arguments_unhashable(self):

diff --git a/Lib/token.py b/Lib/token.py
@@ -63,14 +63,15 @@
 ATEQUAL = 50
 RARROW = 51
 ELLIPSIS = 52
+COLONEQUAL = 53
 # Don't forget to update the table _PyParser_TokenNames in tokenizer.c!
-OP = 53
-ERRORTOKEN = 54
+OP = 54
+ERRORTOKEN = 55
 # These aren't used by the C tokenizer but are needed for tokenize.py
-COMMENT = 55
-NL = 56
-ENCODING = 57
-N_TOKENS = 58
+COMMENT = 56
+NL = 57
+ENCODING = 58
+N_TOKENS = 59
 # Special definitions for cooperation with parser
 NT_OFFSET = 256
 #--end constants--

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
@@ -89,6 +89,7 @@
     '->':  RARROW,
     '@':   AT,
     '@=':  ATEQUAL,
+    ':=':  COLONEQUAL,
 }
 
 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
@@ -169,7 +170,7 @@ def _compile(expr):
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
                  r"//=?", r"->",
-                 r"[+\-*/%&@|^=<>]=?",
+                 r"[+\-*/%&@|^=<>:]=?",
                  r"~")
 
 Bracket = '[][(){}]'

diff --git a/Lib/typing.py b/Lib/typing.py
@@ -165,7 +165,7 @@ def _collect_type_vars(types):
         if isinstance(t, TypeVar) and t not in tvars:
             tvars.append(t)
         if isinstance(t, _GenericAlias) and not t._special:
-            tvars.extend([t for t in t.__parameters__ if t not in tvars])
+            tvars.extend([ty for ty in t.__parameters__ if ty not in tvars])
     return tuple(tvars)
 
 

diff --git a/Parser/Python.asdl b/Parser/Python.asdl
@@ -70,6 +70,7 @@ module Python
          | SetComp(expr elt, comprehension* generators)
          | DictComp(expr key, expr value, comprehension* generators)
          | GeneratorExp(expr elt, comprehension* generators)
+         | AssignExp(expr target, expr value)
          -- the grammar constrains where yield expressions can occur
          | Await(expr value)
          | Yield(expr? value)

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -104,6 +104,7 @@ const char *_PyParser_TokenNames[] = {
     "ATEQUAL",
     "RARROW",
     "ELLIPSIS",
+    "COLONEQUAL",
     /* This table must match the #defines in token.h! */
     "OP",
     "<ERRORTOKEN>",
@@ -1218,6 +1219,11 @@ PyToken_TwoChars(int c1, int c2)
         case '=':               return ATEQUAL;
         }
         break;
+    case ':':
+        switch (c2) {
+        case '=':               return COLONEQUAL;
+        }
+        break;
     }
     return OP;
 }