microsoft · MikhailArkhipov · Feb 16, 2019 · Dec 8, 2018 · Dec 8, 2018 · Dec 8, 2018
diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs
@@ -157,7 +157,7 @@ public async Task<IMember> GetValueFromFunctionTypeAsync(IPythonFunctionType fn,
             if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized ||
                 instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized ||
                 instanceType.Equals(fn.DeclaringType) ||
-                fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].ReturnDocumentation)) {
+                fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation(null))) {
 
                 if (fn.IsSpecialized && fn is PythonFunctionType ft) {
                     foreach (var module in ft.Dependencies) {
@@ -218,7 +218,7 @@ private async Task<ArgumentSet> FindOverloadAsync(IPythonFunctionType fn, IPytho
             var result = noErrorsMatches.Any()
                 ? noErrorsMatches.FirstOrDefault(args => IsMatch(args, fn.Overloads[args.OverloadIndex].Parameters))
                 : null;
-            
+
             // Optimistically pick the best available.
             return result ?? orderedSets.FirstOrDefault();
         }

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs
@@ -31,7 +31,7 @@ public async Task<IMember> GetValueFromIndexAsync(IndexExpression expr, Cancella
             }
 
             var target = await GetValueFromExpressionAsync(expr.Target, cancellationToken);
-            // Try generics
+            // Try generics first since this may be an expression like Dict[int, str]
             var result = await GetValueFromGenericAsync(target, expr, cancellationToken);
             if (result != null) {
                 return result;
@@ -48,7 +48,9 @@ public async Task<IMember> GetValueFromIndexAsync(IndexExpression expr, Cancella
                     instance = new PythonInstance(type);
                 }
                 var index = await GetValueFromExpressionAsync(expr.Index, cancellationToken);
-                return type.Index(instance, index);
+                if (index != null) {
+                    return type.Index(instance, index);
+                }
             }
 
             return UnknownType;
@@ -93,7 +95,7 @@ public async Task<IMember> GetValueFromSetAsync(SetExpression expression, Cancel
 
         public async Task<IMember> GetValueFromGeneratorAsync(GeneratorExpression expression, CancellationToken cancellationToken = default) {
             var iter = expression.Iterators.OfType<ComprehensionFor>().FirstOrDefault();
-            if(iter != null) { 
+            if (iter != null) {
                 return await GetValueFromExpressionAsync(iter.List, cancellationToken) ?? UnknownType;
             }
             return UnknownType;

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Generics.cs
@@ -13,14 +13,14 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
-using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Specializations.Typing.Types;
 using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Core;
 using Microsoft.Python.Parsing.Ast;
 
@@ -47,8 +47,10 @@ private async Task<IMember> GetValueFromGenericAsync(IMember target, Expression
             // rather than specific type instantiation as in List[str].
 
             IPythonType[] specificTypes;
+            var returnInstance = false;
             switch (expr) {
-                case IndexExpression indexExpr: {
+                // Indexing returns type as from A[int]
+                case IndexExpression indexExpr:
                     // Generic[T1, T2, ...] or A[type]()
                     var indices = await EvaluateIndexAsync(indexExpr, cancellationToken);
                     // See which ones are generic parameters as defined by TypeVar() 
@@ -66,6 +68,7 @@ private async Task<IMember> GetValueFromGenericAsync(IMember target, Expression
                     if (genericTypeArgs.Length > 0 && genericTypeArgs.Length != indices.Count) {
                         // TODO: report that some type arguments are not declared with TypeVar.
                     }
+
                     if (specificTypes.Length > 0 && specificTypes.Length != indices.Count) {
                         // TODO: report that arguments are not specific types or are not declared.
                     }
@@ -84,20 +87,21 @@ private async Task<IMember> GetValueFromGenericAsync(IMember target, Expression
 
                         if (specificTypes.Length > 0) {
                             // If target is a generic type and indexes are specific types, create specific class
-                            return await gt.CreateSpecificTypeAsync(new ArgumentSet(specificTypes), Module, GetLoc(expr), cancellationToken);
+                            return gt.CreateSpecificType(new ArgumentSet(specificTypes), Module, GetLoc(expr));
                         } else {
                             // TODO: report too few type arguments for the Generic[].
                             return UnknownType;
                         }
                     }
-
                     break;
-                }
+
                 case CallExpression callExpr:
                     // Alternative instantiation:
                     //  class A(Generic[T]): ...
                     //  x = A(1234)
-                    specificTypes = (await EvaluateICallArgsAsync(callExpr, cancellationToken)).Select(x => x.GetPythonType()).ToArray();
+                    specificTypes = (await EvaluateCallArgsAsync(callExpr, cancellationToken)).Select(x => x.GetPythonType()).ToArray();
+                    // Callable returns instance (as opposed to a type with index expression)
+                    returnInstance = true;
                     break;
 
                 default:
@@ -109,9 +113,12 @@ private async Task<IMember> GetValueFromGenericAsync(IMember target, Expression
             // as we resolve classes on demand. Therefore we don't know if class is generic
             // or not at the time of the PythonClassType creation.
             // TODO: figure out if we could make GenericClassType: PythonClassType, IGenericType instead.
-            return target is PythonClassType cls 
-                ? await cls.CreateSpecificTypeAsync(new ArgumentSet(specificTypes), Module, GetLoc(expr), cancellationToken) 
-                : null;
+            if (target is PythonClassType cls) {
+                var location = GetLoc(expr);
+                var type = cls.CreateSpecificType(new ArgumentSet(specificTypes), Module, location);
+                return returnInstance ? new PythonInstance(type, GetLoc(expr)) : (IMember)type;
+            }
+            return null;
         }
 
         private async Task<IReadOnlyList<IMember>> EvaluateIndexAsync(IndexExpression expr, CancellationToken cancellationToken = default) {
@@ -124,16 +131,16 @@ private async Task<IReadOnlyList<IMember>> EvaluateIndexAsync(IndexExpression ex
                 }
             } else {
                 var index = await GetValueFromExpressionAsync(expr.Index, cancellationToken);
-                indices.Add(index);
+                indices.Add(index ?? UnknownType);
             }
             return indices;
         }
 
-        private async Task<IReadOnlyList<IMember>> EvaluateICallArgsAsync(CallExpression expr, CancellationToken cancellationToken = default) {
+        private async Task<IReadOnlyList<IMember>> EvaluateCallArgsAsync(CallExpression expr, CancellationToken cancellationToken = default) {
             var indices = new List<IMember>();
             cancellationToken.ThrowIfCancellationRequested();
-            foreach (var e in expr.Args.Select(a => a.Expression).ExcludeDefault()) {
-                var value = await GetValueFromExpressionAsync(e, cancellationToken);
+            foreach (var e in expr.Args.Select(a => a.Expression)) {
+                var value = await GetValueFromExpressionAsync(e, cancellationToken) ?? UnknownType;
                 indices.Add(value);
             }
             return indices;

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs
@@ -39,7 +39,7 @@ private async Task<IMember> GetValueFromUnaryOpAsync(UnaryExpression expr, Cance
                 case PythonOperator.Pos:
                     return await GetValueFromUnaryOpAsync(expr, "__pos__", cancellationToken);
             }
-            return null;
+            return UnknownType;
         }
 
         private async Task<IMember> GetValueFromUnaryOpAsync(UnaryExpression expr, string op, CancellationToken cancellationToken = default) {
@@ -57,7 +57,7 @@ private async Task<IMember> GetValueFromUnaryOpAsync(UnaryExpression expr, strin
                     ? new PythonConstant(-value, c.Type, GetLoc(expr))
                     : instance;
             }
-            return null;
+            return UnknownType;
         }
 
         private async Task<IMember> GetValueFromBinaryOpAsync(Expression expr, CancellationToken cancellationToken = default) {
@@ -87,8 +87,8 @@ private async Task<IMember> GetValueFromBinaryOpAsync(Expression expr, Cancellat
                     return Interpreter.GetBuiltinType(BuiltinTypeId.Bool);
             }
 
-            var left = await GetValueFromExpressionAsync(binop.Left, cancellationToken);
-            var right = await GetValueFromExpressionAsync(binop.Right, cancellationToken);
+            var left = await GetValueFromExpressionAsync(binop.Left, cancellationToken) ?? UnknownType;
+            var right = await GetValueFromExpressionAsync(binop.Right, cancellationToken) ?? UnknownType;
 
             switch (binop.Operator) {
                 case PythonOperator.Divide:

diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs
@@ -192,7 +192,7 @@ private async Task<IMember> GetValueFromMemberAsync(MemberExpression expr, Cance
             }
 
             instance = instance ?? m as IPythonInstance;
-            var type = m.GetPythonType(); // Try inner type
+            var type = m?.GetPythonType(); // Try inner type
             var value = type?.GetMember(expr.Name);
 
             // Class type GetMember returns a type. However, class members are
@@ -226,7 +226,7 @@ private async Task<IMember> GetValueFromConditionalAsync(ConditionalExpression e
             var trueValue = await GetValueFromExpressionAsync(expr.TrueExpression, cancellationToken);
             var falseValue = await GetValueFromExpressionAsync(expr.FalseExpression, cancellationToken);
 
-            return trueValue ?? falseValue;
+            return trueValue ?? falseValue ?? UnknownType;
         }
 
         public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) {

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs
@@ -32,7 +32,7 @@ public async Task HandleAssignmentAsync(AssignmentStatement node, CancellationTo
                 return;
             }
 
-            var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken);
+            var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken) ?? Eval.UnknownType;
             // Check PEP hint first
             var valueType = Eval.GetTypeFromPepHint(node.Right);
             if (valueType != null) {

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/TryExceptHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/TryExceptHandler.cs
@@ -28,7 +28,7 @@ public async Task<bool> HandleTryExceptAsync(TryStatement node, CancellationToke
             foreach (var handler in node.Handlers.MaybeEnumerate()) {
                 if (handler.Test != null && handler.Target is NameExpression nex) {
                     var value = await Eval.GetValueFromExpressionAsync(handler.Test, cancellationToken);
-                    Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, nex);
+                    Eval.DeclareVariable(nex.Name, value ?? Eval.UnknownType, VariableSource.Declaration, nex);
                 }
                 await handler.Body.WalkAsync(Walker, cancellationToken);
             }

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs
@@ -22,8 +22,8 @@
 using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis.Analyzer.Handlers {
-    internal sealed class TupleExpressionHandler: StatementHandler {
-        public TupleExpressionHandler(AnalysisWalker walker): base(walker) { }
+    internal sealed class TupleExpressionHandler : StatementHandler {
+        public TupleExpressionHandler(AnalysisWalker walker) : base(walker) { }
 
         public async Task HandleTupleAssignmentAsync(TupleExpression lhs, Expression rhs, IMember value, CancellationToken cancellationToken = default) {
             cancellationToken.ThrowIfCancellationRequested();
@@ -34,9 +34,7 @@ public async Task HandleTupleAssignmentAsync(TupleExpression lhs, Expression rhs
                 for (var i = 0; i < Math.Min(names.Length, returnedExpressions.Length); i++) {
                     if (returnedExpressions[i] != null && !string.IsNullOrEmpty(names[i])) {
                         var v = await Eval.GetValueFromExpressionAsync(returnedExpressions[i], cancellationToken);
-                        if (v != null) {
-                            Eval.DeclareVariable(names[i], v, VariableSource.Declaration, returnedExpressions[i]);
-                        }
+                        Eval.DeclareVariable(names[i], v ?? Eval.UnknownType, VariableSource.Declaration, returnedExpressions[i]);
                     }
                 }
                 return;

diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/WithHandler.cs
@@ -28,7 +28,7 @@ public async Task HandleWithAsync(WithStatement node, CancellationToken cancella
             cancellationToken.ThrowIfCancellationRequested();
             foreach (var item in node.Items.Where(x => x.Variable != null)) {
                 var contextManager = await Eval.GetValueFromExpressionAsync(item.ContextManager, cancellationToken);
-                var cmType = contextManager.GetPythonType();
+                var cmType = contextManager?.GetPythonType();
 
                 var enter = cmType?.GetMember(node.IsAsync ? @"__aenter__" : @"__enter__")?.GetPythonType<IPythonFunctionType>();
                 if (enter != null) {

diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs
@@ -87,7 +87,7 @@ public override async Task EvaluateAsync(CancellationToken cancellationToken = d
         }
 
         public override async Task<bool> WalkAsync(AssignmentStatement node, CancellationToken cancellationToken = default) {
-            var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken);
+            var value = await Eval.GetValueFromExpressionAsync(node.Right, cancellationToken) ?? Eval.UnknownType;
 
             foreach (var lhs in node.Left) {
                 switch (lhs) {
@@ -180,7 +180,7 @@ private async Task DeclareParameterAsync(Parameter p, int index, ParameterInfo p
             } else {
                 var defaultValue = await Eval.GetValueFromExpressionAsync(p.DefaultValue, cancellationToken) ?? Eval.UnknownType;
 
-                paramType = defaultValue.GetPythonType();
+                paramType = defaultValue?.GetPythonType();
                 if (!paramType.IsUnknown()) {
                     pi?.SetDefaultValueType(paramType);
                 }

diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
@@ -54,7 +54,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana
             // 'type()' in code is a function call, not a type class instantiation.
             if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) {
                 f = PythonFunctionType.ForSpecialization(name, analysis.Document);
-                f.AddOverload(new PythonFunctionOverload(name, analysis.Document, LocationInfo.Empty));
+                f.AddOverload(new PythonFunctionOverload(name, analysis.Document, _ => LocationInfo.Empty));
                 analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration, LocationInfo.Empty);
             }
             return f;

diff --git a/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs b/src/Analysis/Ast/Impl/Extensions/MemberExtensions.cs
@@ -16,7 +16,6 @@
 using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
-using Microsoft.Python.Parsing.Ast;
 
 namespace Microsoft.Python.Analysis {
     public static class MemberExtensions {

diff --git a/src/Analysis/Ast/Impl/Specializations/Specialized.cs b/src/Analysis/Ast/Impl/Specializations/Specialized.cs
@@ -19,15 +19,15 @@ namespace Microsoft.Python.Analysis.Specializations {
     internal static class Specialized {
         public static IPythonPropertyType Property(string name, IPythonModule declaringModule, IPythonType declaringType, IMember returnValue) {
             var prop = new PythonPropertyType(name, declaringModule, declaringType, false, LocationInfo.Empty);
-            var o = new PythonFunctionOverload(prop.Name, declaringModule, LocationInfo.Empty);
+            var o = new PythonFunctionOverload(prop.Name, declaringModule, _ => LocationInfo.Empty);
             o.AddReturnValue(returnValue);
             prop.AddOverload(o);
             return prop;
         }
 
         public static IPythonFunctionType Function(string name, IPythonModule declaringModule, IPythonType declaringType, string documentation, IMember returnValue) {
             var prop = new PythonFunctionType(name, declaringModule, declaringType, documentation, LocationInfo.Empty);
-            var o = new PythonFunctionOverload(prop.Name, declaringModule, LocationInfo.Empty);
+            var o = new PythonFunctionOverload(prop.Name, declaringModule, _ => LocationInfo.Empty);
             o.AddReturnValue(returnValue);
             prop.AddOverload(o);
             return prop;

diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBaseType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Definitions/IGenericClassBaseType.cs
@@ -14,12 +14,13 @@
 // permissions and limitations under the License.
 
 using System.Collections.Generic;
+using Microsoft.Python.Analysis.Types;
 
 namespace Microsoft.Python.Analysis.Specializations.Typing {
     /// <summary>
     /// Represents Generic[T1, T2, ...]. Used as a base class to generic classes.
     /// </summary>
-    public interface IGenericClassBaseType {
+    public interface IGenericClassBaseType: IPythonType {
         IReadOnlyList<IGenericTypeParameter> TypeArgs { get; }
     }
 }
diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBaseType.cs b/src/Analysis/Ast/Impl/Specializations/Typing/Types/GenericClassBaseType.cs
@@ -17,6 +17,9 @@
 using Microsoft.Python.Analysis.Types;
 
 namespace Microsoft.Python.Analysis.Specializations.Typing.Types {
+    /// <summary>
+    /// Represents Generic[T1, T2, ...]
+    /// </summary>
     internal sealed class GenericClassBaseType: PythonClassType, IGenericClassBaseType {
         internal GenericClassBaseType(IReadOnlyList<IGenericTypeParameter> typeArgs, IPythonModule declaringModule, LocationInfo location) 
             : base("Generic", declaringModule, location) {