microsoft · MikhailArkhipov · Mar 14, 2019 · Mar 1, 2019 · Mar 1, 2019 · Mar 4, 2019
diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs
@@ -62,23 +62,24 @@ public override bool Walk(AssignmentStatement node) {
         }
 
         public override bool Walk(ExpressionStatement node) {
-            AssignmentHandler.HandleAnnotatedExpression(node.Expression as ExpressionWithAnnotation, null);
+            switch (node.Expression) {
+                case ExpressionWithAnnotation ea:
+                    AssignmentHandler.HandleAnnotatedExpression(ea, null);
+                    break;
+                case Comprehension comp:
+                    Eval.ProcessComprehension(comp);
+                    break;
+            }
             return false;
         }
 
-        public override bool Walk(ForStatement node) {
-            LoopHandler.HandleFor(node);
-            return base.Walk(node);
-        }
-
+        public override bool Walk(ForStatement node) => LoopHandler.HandleFor(node);
         public override bool Walk(FromImportStatement node) => ImportHandler.HandleFromImport(node);
         public override bool Walk(GlobalStatement node) => NonLocalHandler.HandleGlobal(node);
         public override bool Walk(IfStatement node) => ConditionalHandler.HandleIf(node);
 
         public override bool Walk(ImportStatement node) => ImportHandler.HandleImport(node);
-
-        public override bool Walk(NonlocalStatement node)
-            => NonLocalHandler.HandleNonLocal(node);
+        public override bool Walk(NonlocalStatement node) => NonLocalHandler.HandleNonLocal(node);
 
         public override bool Walk(TryStatement node) {
             TryExceptHandler.HandleTryExcept(node);

diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs
@@ -15,6 +15,7 @@
 
 using System;
 using System.Collections.Generic;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Diagnostics;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
@@ -57,7 +58,7 @@ public interface IExpressionEvaluator {
         /// </summary>
         IMember GetValueFromExpression(Expression expr);
 
-        IMember LookupNameInScopes(string name, out IScope scope);
+        IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options = LookupOptions.Normal);
 
         IPythonType GetTypeFromString(string typeString);
 

diff --git a/...Impl/Analyzer/Evaluation/LookupOptions.cs → ...mpl/Analyzer/Definitions/LookupOptions.cs b/...Impl/Analyzer/Evaluation/LookupOptions.cs → ...mpl/Analyzer/Definitions/LookupOptions.cs
@@ -15,9 +15,9 @@
 
 using System;
 
-namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
+namespace Microsoft.Python.Analysis.Analyzer {
     [Flags]
-    internal enum LookupOptions {
+    public enum LookupOptions {
         None = 0,
         Local,
         Nonlocal,

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs
@@ -69,6 +69,18 @@ public IMember GetValueFromCallable(CallExpression expr) {
             return value;
         }
 
+        public IMember GetValueFromLambda(LambdaExpression expr) {
+            if (expr == null) {
+                return null;
+            }
+
+            var loc = GetLoc(expr);
+            var ft = new PythonFunctionType(expr.Function, Module, null, loc);
+            var overload = new PythonFunctionOverload(expr.Function, ft, Module, GetLoc(expr));
+            ft.AddOverload(overload);
+            return ft;
+        }
+
         public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) {
             SymbolTable.Evaluate(cls.ClassDefinition);
             // Determine argument types

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs
@@ -98,5 +98,65 @@ public IMember GetValueFromGenerator(GeneratorExpression expression) {
             }
             return UnknownType;
         }
+
+        public IMember GetValueFromComprehension(Comprehension node) {
+            var oldVariables = CurrentScope.Variables.OfType<Variable>().ToDictionary(k => k.Name, v => v);
+            try {
+                ProcessComprehension(node);
+                switch (node) {
+                    case ListComprehension lc:
+                        var v1 = GetValueFromExpression(lc.Item) ?? UnknownType;
+                        return PythonCollectionType.CreateList(Interpreter, GetLoc(lc), new[] { v1 });
+                    case SetComprehension sc:
+                        var v2 = GetValueFromExpression(sc.Item) ?? UnknownType;
+                        return PythonCollectionType.CreateSet(Interpreter, GetLoc(sc), new[] { v2 });
+                    case DictionaryComprehension dc:
+                        var k = GetValueFromExpression(dc.Key) ?? UnknownType;
+                        var v = GetValueFromExpression(dc.Value) ?? UnknownType;
+                        return new PythonDictionary(new PythonDictionaryType(Interpreter), GetLoc(dc), new Dictionary<IMember, IMember> { { k, v } });
+                }
+
+                return UnknownType;
+            } finally {
+                // Remove temporary variables since this is assignment and the right hand
+                // side comprehension does not leak internal variables into the scope.
+                var newVariables = CurrentScope.Variables.ToDictionary(k => k.Name, v => v);
+                var variables = (VariableCollection)CurrentScope.Variables;
+                foreach (var kvp in newVariables) {
+                    if (!oldVariables.ContainsKey(kvp.Key)) {
+                        variables.RemoveVariable(kvp.Key);
+                    } else {
+                        variables.DeclareVariable(oldVariables[kvp.Key]);
+                    }
+                }
+            }
+        }
+
+        internal void ProcessComprehension(Comprehension node) {
+            foreach (var cfor in node.Iterators.OfType<ComprehensionFor>().Where(c => c.Left != null)) {
+                var value = GetValueFromExpression(cfor.List);
+                if (value != null) {
+                    switch (cfor.Left) {
+                        case NameExpression nex when value is IPythonCollection coll:
+                            DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, GetLoc(nex));
+                            break;
+                        case NameExpression nex:
+                            DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, GetLoc(nex));
+                            break;
+                        case TupleExpression tex when value is IPythonDictionary dict && tex.Items.Count > 0:
+                            if (tex.Items[0] is NameExpression nx0 && !string.IsNullOrEmpty(nx0.Name)) {
+                                DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx0));
+                            }
+                            if (tex.Items.Count > 1 && tex.Items[1] is NameExpression nx1 && !string.IsNullOrEmpty(nx1.Name)) {
+                                DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx1));
+                            }
+                            foreach (var item in tex.Items.Skip(2).OfType<NameExpression>().Where(x => !string.IsNullOrEmpty(x.Name))) {
+                                DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, GetLoc(item));
+                            }
+                            break;
+                    }
+                }
+            }
+        }
     }
 }
diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs
@@ -141,8 +141,8 @@ public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, Lo
         /// as a child of the specified scope. Scope is pushed on the stack
         /// and will be removed when returned the disposable is disposed.
         /// </summary>
-        public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope fromScope) {
-            fromScope = null;
+        public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope outerScope) {
+            outerScope = null;
             if (node == null) {
                 return Disposable.Empty;
             }
@@ -156,25 +156,25 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop
             }
 
             if (node.Parent != null) {
-                if (!_scopeLookupCache.TryGetValue(node.Parent, out fromScope)) {
-                    fromScope = gs
+                if (!_scopeLookupCache.TryGetValue(node.Parent, out outerScope)) {
+                    outerScope = gs
                         .TraverseDepthFirst(s => s.Children.OfType<Scope>())
                         .FirstOrDefault(s => s.Node == node.Parent);
-                    _scopeLookupCache[node.Parent] = fromScope;
+                    _scopeLookupCache[node.Parent] = outerScope;
                 }
             }
 
-            fromScope = fromScope ?? gs;
-            if (fromScope != null) {
+            outerScope = outerScope ?? gs;
+            if (outerScope != null) {
                 Scope scope;
                 if (node is PythonAst) {
                     // node points to global scope, it is not a function or a class.
                     scope = gs;
                 } else {
-                    scope = fromScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
+                    scope = outerScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
                     if (scope == null) {
-                        scope = new Scope(node, fromScope, true);
-                        fromScope.AddChildScope(scope);
+                        scope = new Scope(node, outerScope, true);
+                        outerScope.AddChildScope(scope);
                         _scopeLookupCache[node] = scope;
                     }
                 }

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs
@@ -69,6 +69,15 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
         public LocationInfo GetLocation(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty;
         public IEnumerable<DiagnosticsEntry> Diagnostics => _diagnostics;
 
+        public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) {
+            // Do not add if module is library, etc. Only handle user code.
+            if (Module.ModuleType == ModuleType.User) {
+                lock (_lock) {
+                    _diagnostics.Add(entry);
+                }
+            }
+        }
+
         public IMember GetValueFromExpression(Expression expr)
             => GetValueFromExpression(expr, DefaultLookupOptions);
 
@@ -89,9 +98,7 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) {
                 return null;
             }
 
-            while (expr is ParenthesisExpression parExpr) {
-                expr = parExpr.Expression;
-            }
+            expr = expr.RemoveParenthesis();
 
             IMember m;
             switch (expr) {
@@ -131,6 +138,12 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) {
                 case GeneratorExpression genex:
                     m = GetValueFromGenerator(genex);
                     break;
+                case Comprehension comp:
+                    m = GetValueFromComprehension(comp);
+                    break;
+                case LambdaExpression lambda:
+                    m = GetValueFromLambda(lambda);
+                    break;
                 default:
                     m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options);
                     break;
@@ -228,14 +241,5 @@ private IMember GetValueFromConditional(ConditionalExpression expr) {
 
             return trueValue ?? falseValue ?? UnknownType;
         }
-
-        public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) {
-            // Do not add if module is library, etc. Only handle user code.
-            if (Module.ModuleType == ModuleType.User) {
-                lock (_lock) {
-                    _diagnostics.Add(entry);
-                }
-            }
-        }
     }
 }
diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs
@@ -15,7 +15,6 @@
 
 using System.Diagnostics;
 using System.Linq;
-using Microsoft.Python.Analysis.Analyzer.Evaluation;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
@@ -62,7 +61,7 @@ public void HandleAssignment(AssignmentStatement node) {
                 if (Eval.CurrentScope.NonLocals[ne.Name] != null) {
                     Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal);
                     if (scope != null) {
-                        scope.Variables[ne.Name].Value = value;
+                        scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne));
                     } else {
                         // TODO: report variable is not declared in outer scopes.
                     }
@@ -72,7 +71,7 @@ public void HandleAssignment(AssignmentStatement node) {
                 if (Eval.CurrentScope.Globals[ne.Name] != null) {
                     Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global);
                     if (scope != null) {
-                        scope.Variables[ne.Name].Value = value;
+                        scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne));
                     } else {
                         // TODO: report variable is not declared in global scope.
                     }

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs
@@ -15,8 +15,6 @@
 
 using System;
 using System.Linq;
-using Microsoft.Python.Analysis.Types;
-using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core.OS;
 using Microsoft.Python.Parsing;
 using Microsoft.Python.Parsing.Ast;

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs
@@ -21,35 +21,27 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers {
     internal sealed class LoopHandler : StatementHandler {
         public LoopHandler(AnalysisWalker walker) : base(walker) { }
 
-        public void HandleFor(ForStatement node) {
+        public bool HandleFor(ForStatement node) {
             var iterable = Eval.GetValueFromExpression(node.List);
             var iterator = (iterable as IPythonIterable)?.GetIterator();
-            if (iterator == null) {
-                // TODO: report that expression does not evaluate to iterable.
-            }
             var value = iterator?.Next ?? Eval.UnknownType;
-
             switch (node.Left) {
                 case NameExpression nex:
                     // for x in y:
-                    Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(node.Left));
+                    if (!string.IsNullOrEmpty(nex.Name)) {
+                        Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(nex));
+                    }
                     break;
                 case TupleExpression tex:
                     // x = [('abc', 42, True), ('abc', 23, False)]
-                    // for some_str, some_int, some_bool in x:
-                    var names = tex.Items.OfType<NameExpression>().Select(x => x.Name).ToArray();
-                    if (value is IPythonIterable valueIterable) {
-                        var valueIterator = valueIterable.GetIterator();
-                        foreach (var n in names) {
-                            Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, VariableSource.Declaration, Eval.GetLoc(node.Left));
-                        }
-                    } else {
-                        // TODO: report that expression yields value that does not evaluate to iterable.
-                    }
+                    // for some_str, (some_int, some_bool) in x:
+                    var h = new TupleExpressionHandler(Walker);
+                    h.HandleTupleAssignment(tex, node.List, value);
                     break;
             }
 
             node.Body?.Walk(Walker);
+            return false;
         }
 
         public void HandleWhile(WhileStatement node) {

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs
@@ -14,10 +14,8 @@
 // permissions and limitations under the License.
 
 using Microsoft.Python.Analysis.Analyzer.Evaluation;
-using Microsoft.Python.Analysis.Analyzer.Symbols;
 using Microsoft.Python.Analysis.Modules;
 using Microsoft.Python.Analysis.Types;
-using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core.Logging;
 using Microsoft.Python.Parsing.Ast;