Skip to content

WIP: Demo: Offer Tokenizing Improvements for Highlighting #10295

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
msftrncs wants to merge 2 commits into
base: master
Choose a base branch
from
Open

WIP: Demo: Offer Tokenizing Improvements for Highlighting #10295

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a1f2ec6
Offer Tokenizing Improvements for Highlighting
msftrncs Aug 3, 2019
6fd8322
Address `--` and `--%` highlight issues
msftrncs Sep 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 41 additions & 26 deletions src/System.Management.Automation/engine/parser/Parser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5282,7 +5282,14 @@ private StatementAst FunctionDeclarationRule(Token functionToken)
}

// A function name that matches a keyword isn't really a keyword, so don't color it that way
functionNameToken.TokenFlags &= ~TokenFlags.Keyword;
functionNameToken.SetIsCommandArgument();
//functionNameToken.TokenFlags &= ~TokenFlags.Keyword;

// A function name is not an expandable token, so remove any nested tokens
if (functionNameToken is StringExpandableToken expandableToken)
{
expandableToken.NestedTokens = null;
}

IScriptExtent endErrorStatement;
Token rParen;
Expand Down Expand Up @@ -5317,8 +5324,8 @@ private StatementAst FunctionDeclarationRule(Token functionToken)
_tokenizer.InWorkflowContext = isWorkflow;

ScriptBlockAst scriptBlock = ScriptBlockRule(lCurly, isFilter);
var functionName = (functionNameToken.Kind == TokenKind.Generic)
? ((StringToken)functionNameToken).Value
var functionName = (functionNameToken is StringToken stringFunctionNameToken)
? stringFunctionNameToken.Value
: functionNameToken.Text;

FunctionDefinitionAst result = new FunctionDefinitionAst(ExtentOf(functionToken, scriptBlock),
Expand Down Expand Up @@ -6049,7 +6056,7 @@ private ExpressionAst GetCommandArgument(CommandArgumentContext context, Token t
case TokenKind.AndAnd:
case TokenKind.OrOr:
case TokenKind.Ampersand:
case TokenKind.MinusMinus:
// case TokenKind.MinusMinus when (context == CommandArgumentContext.CommandName):
case TokenKind.Comma:
UngetToken(token);

Expand Down Expand Up @@ -6103,10 +6110,16 @@ private ExpressionAst GetCommandArgument(CommandArgumentContext context, Token t
}
else
{
// Remove any nested tokens already collected in this context
if (expandableToken != null && context == CommandArgumentContext.CommandName)
{
expandableToken.NestedTokens = null;
}

exprAst = new StringConstantExpressionAst(genericToken.Extent, genericToken.Value, StringConstantType.BareWord);

// If this is a verbatim argument, then don't continue peeking
if (string.Equals(genericToken.Value, VERBATIM_ARGUMENT, StringComparison.OrdinalIgnoreCase))
if (context == CommandArgumentContext.CommandArgument && string.Equals(genericToken.Value, VERBATIM_ARGUMENT, StringComparison.OrdinalIgnoreCase))
{
foundVerbatimArgument = true;
}
Expand All @@ -6118,7 +6131,8 @@ private ExpressionAst GetCommandArgument(CommandArgumentContext context, Token t
exprAst = new StringConstantExpressionAst(token.Extent, token.Text, StringConstantType.BareWord);

// A command/argument that matches a keyword isn't really a keyword, so don't color it that way
token.TokenFlags &= ~TokenFlags.Keyword;
// token.TokenFlags &= ~TokenFlags.Keyword;
token.SetIsCommandArgument();

switch (context)
{
Expand All @@ -6129,7 +6143,6 @@ private ExpressionAst GetCommandArgument(CommandArgumentContext context, Token t
case CommandArgumentContext.FileName:
case CommandArgumentContext.CommandArgument:
case CommandArgumentContext.SwitchCondition:
token.SetIsCommandArgument();
break;
}

Expand Down Expand Up @@ -6260,13 +6273,11 @@ internal Ast CommandRule(bool forDynamicKeyword)
scanning = false;
continue;

case TokenKind.MinusMinus:
case TokenKind.MinusMinus when !sawDashDash && context != CommandArgumentContext.CommandNameAfterInvocationOperator:
endExtent = token.Extent;
// Add the first -- as a parameter, which is then ignored when constructing the command processor unless it's a native
// command. All subsequent -- are added as arguments.
elements.Add(sawDashDash
? (CommandElementAst)new StringConstantExpressionAst(token.Extent, token.Text, StringConstantType.BareWord)
: new CommandParameterAst(token.Extent, "-", null, token.Extent));
elements.Add(new CommandParameterAst(token.Extent, "-", null, token.Extent));
sawDashDash = true;
break;

Expand All @@ -6278,8 +6289,8 @@ internal Ast CommandRule(bool forDynamicKeyword)
SkipNewlines();
break;

case TokenKind.Parameter:
if ((context & CommandArgumentContext.CommandName) != 0 || sawDashDash)
case TokenKind.Parameter when !sawDashDash && (context & CommandArgumentContext.CommandName) == 0:
if ((context & CommandArgumentContext.CommandName) != 0)
{
endExtent = token.Extent;
token.TokenFlags |= TokenFlags.CommandName;
Expand Down Expand Up @@ -6339,6 +6350,7 @@ internal Ast CommandRule(bool forDynamicKeyword)
// but V3 and on will because it falls out rather naturally here.
endExtent = token.Extent;
elements.Add(new StringConstantExpressionAst(token.Extent, token.Text, StringConstantType.BareWord));
token.TokenFlags |= TokenFlags.CommandName;
}

break;
Expand All @@ -6356,24 +6368,27 @@ internal Ast CommandRule(bool forDynamicKeyword)
{
var ast = GetCommandArgument(context, token);

// If this is the special verbatim argument syntax, look for the next element
StringToken argumentToken = token as StringToken;
if ((argumentToken != null) && string.Equals(argumentToken.Value, VERBATIM_ARGUMENT, StringComparison.OrdinalIgnoreCase))
if (context == CommandArgumentContext.CommandArgument)
{
elements.Add(ast);
endExtent = ast.Extent;

var verbatimToken = GetVerbatimCommandArgumentToken();
if (verbatimToken != null)
// If this is the special verbatim argument syntax, look for the next element
StringToken argumentToken = token as StringToken;
if ((argumentToken != null) && string.Equals(argumentToken.Value, VERBATIM_ARGUMENT, StringComparison.OrdinalIgnoreCase))
{
foundVerbatimArgument = true;
scanning = false;
ast = new StringConstantExpressionAst(verbatimToken.Extent, verbatimToken.Value, StringConstantType.BareWord);
elements.Add(ast);
endExtent = ast.Extent;
}

break;
var verbatimToken = GetVerbatimCommandArgumentToken();
if (verbatimToken != null)
{
foundVerbatimArgument = true;
scanning = false;
ast = new StringConstantExpressionAst(verbatimToken.Extent, verbatimToken.Value, StringConstantType.BareWord);
elements.Add(ast);
endExtent = ast.Extent;
}

break;
}
}

endExtent = ast.Extent;
Expand Down
2 changes: 1 addition & 1 deletion src/System.Management.Automation/engine/parser/ast.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3618,7 +3618,7 @@ internal FunctionDefinitionAst(IScriptExtent extent,
: this(extent,
isFilter,
isWorkflow,
(functionNameToken.Kind == TokenKind.Generic) ? ((StringToken)functionNameToken).Value : functionNameToken.Text,
(functionNameToken is StringToken stringFunctionNameToken) ? stringFunctionNameToken.Value : functionNameToken.Text,
parameters,
body)
{
Expand Down
1 change: 1 addition & 0 deletions src/System.Management.Automation/engine/parser/token.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1203,6 +1203,7 @@ internal void SetIsCommandArgument()
// considered a command argument. This prevent arbitrary changes to the kind which should be safer.
if (_kind != TokenKind.Identifier)
{
_tokenFlags = _tokenFlags & ~_kind.GetTraits() | TokenKind.Generic.GetTraits();
_kind = TokenKind.Generic;
}
}
Expand Down