angular · petebacondarwin · Feb 8, 2019 · Feb 8, 2019 · Feb 8, 2019
diff --git a/packages/compiler-cli/test/compliance/r3_view_compiler_i18n_spec.ts b/packages/compiler-cli/test/compliance/r3_view_compiler_i18n_spec.ts
@@ -41,7 +41,8 @@ const extract = (from: string, regex: any, transformFn: (match: any[]) => any) =
 const verifyTranslationIds =
     (source: string, output: string, exceptions = {},
      interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG) => {
-      const parseResult = htmlParser.parse(source, 'path:://to/template', true);
+      const parseResult =
+          htmlParser.parse(source, 'path:://to/template', {tokenizeExpansionForms: true});
       const extractedIdToMsg = new Map<string, any>();
       const extractedIds = new Set<string>();
       const generatedIds = new Set<string>();

diff --git a/packages/compiler-cli/test/diagnostics/mocks.ts b/packages/compiler-cli/test/diagnostics/mocks.ts
@@ -157,12 +157,7 @@ export class DiagnosticContext {
       };
       const urlResolver = createOfflineCompileUrlResolver();
       const htmlParser = new class extends HtmlParser {
-        parse(
-            source: string, url: string, parseExpansionForms: boolean = false,
-            interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG):
-            ParseTreeResult {
-          return new ParseTreeResult([], []);
-        }
+        parse(): ParseTreeResult { return new ParseTreeResult([], []); }
       };
 
       // This tracks the CompileConfig in codegen.ts. Currently these options
@@ -209,7 +204,7 @@ function compileTemplate(context: DiagnosticContext, type: StaticSymbol, templat
     const parser = new TemplateParser(
         config, context.reflector, expressionParser, new DomElementSchemaRegistry(), htmlParser,
         null !, []);
-    const htmlResult = htmlParser.parse(template, '', true);
+    const htmlResult = htmlParser.parse(template, '', {tokenizeExpansionForms: true});
     const analyzedModules = context.analyzedModules;
     // let errors: Diagnostic[]|undefined = undefined;
     let ngModule = analyzedModules.ngModuleByPipeOrDirective.get(type);

diff --git a/packages/compiler/src/directive_normalizer.ts b/packages/compiler/src/directive_normalizer.ts
@@ -113,12 +113,11 @@ export class DirectiveNormalizer {
       templateAbsUrl: string): PreparsedTemplate {
     const isInline = !!prenormData.template;
     const interpolationConfig = InterpolationConfig.fromArray(prenormData.interpolation !);
+    const templateUrl = templateSourceUrl(
+        {reference: prenormData.ngModuleType}, {type: {reference: prenormData.componentType}},
+        {isInline, templateUrl: templateAbsUrl});
     const rootNodesAndErrors = this._htmlParser.parse(
-        template,
-        templateSourceUrl(
-            {reference: prenormData.ngModuleType}, {type: {reference: prenormData.componentType}},
-            {isInline, templateUrl: templateAbsUrl}),
-        true, interpolationConfig);
+        template, templateUrl, {tokenizeExpansionForms: true, interpolationConfig});
     if (rootNodesAndErrors.errors.length > 0) {
       const errorString = rootNodesAndErrors.errors.join('\n');
       throw syntaxError(`Template parse errors:\n${errorString}`);

diff --git a/packages/compiler/src/i18n/i18n_html_parser.ts b/packages/compiler/src/i18n/i18n_html_parser.ts
@@ -8,7 +8,8 @@
 
 import {MissingTranslationStrategy} from '../core';
 import {HtmlParser} from '../ml_parser/html_parser';
-import {DEFAULT_INTERPOLATION_CONFIG, InterpolationConfig} from '../ml_parser/interpolation_config';
+import {DEFAULT_INTERPOLATION_CONFIG} from '../ml_parser/interpolation_config';
+import {TokenizeOptions} from '../ml_parser/lexer';
 import {ParseTreeResult} from '../ml_parser/parser';
 import {Console} from '../util';
 
@@ -41,11 +42,9 @@ export class I18NHtmlParser implements HtmlParser {
     }
   }
 
-  parse(
-      source: string, url: string, parseExpansionForms: boolean = false,
-      interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ParseTreeResult {
-    const parseResult =
-        this._htmlParser.parse(source, url, parseExpansionForms, interpolationConfig);
+  parse(source: string, url: string, options: TokenizeOptions = {}): ParseTreeResult {
+    const interpolationConfig = options.interpolationConfig || DEFAULT_INTERPOLATION_CONFIG;
+    const parseResult = this._htmlParser.parse(source, url, {interpolationConfig, ...options});
 
     if (parseResult.errors.length) {
       return new ParseTreeResult(parseResult.rootNodes, parseResult.errors);

diff --git a/packages/compiler/src/i18n/message_bundle.ts b/packages/compiler/src/i18n/message_bundle.ts
@@ -27,7 +27,8 @@ export class MessageBundle {
 
   updateFromTemplate(html: string, url: string, interpolationConfig: InterpolationConfig):
       ParseError[] {
-    const htmlParserResult = this._htmlParser.parse(html, url, true, interpolationConfig);
+    const htmlParserResult =
+        this._htmlParser.parse(html, url, {tokenizeExpansionForms: true, interpolationConfig});
 
     if (htmlParserResult.errors.length) {
       return htmlParserResult.errors;

diff --git a/packages/compiler/src/i18n/serializers/xliff.ts b/packages/compiler/src/i18n/serializers/xliff.ts
@@ -185,7 +185,7 @@ class XliffParser implements ml.Visitor {
     this._unitMlString = null;
     this._msgIdToHtml = {};
 
-    const xml = new XmlParser().parse(xliff, url, false);
+    const xml = new XmlParser().parse(xliff, url);
 
     this._errors = xml.errors;
     ml.visitAll(this, xml.rootNodes, null);
@@ -268,7 +268,7 @@ class XmlToI18n implements ml.Visitor {
   private _errors !: I18nError[];
 
   convert(message: string, url: string) {
-    const xmlIcu = new XmlParser().parse(message, url, true);
+    const xmlIcu = new XmlParser().parse(message, url, {tokenizeExpansionForms: true});
     this._errors = xmlIcu.errors;
 
     const i18nNodes = this._errors.length > 0 || xmlIcu.rootNodes.length == 0 ?

diff --git a/packages/compiler/src/i18n/serializers/xliff2.ts b/packages/compiler/src/i18n/serializers/xliff2.ts
@@ -203,7 +203,7 @@ class Xliff2Parser implements ml.Visitor {
     this._unitMlString = null;
     this._msgIdToHtml = {};
 
-    const xml = new XmlParser().parse(xliff, url, false);
+    const xml = new XmlParser().parse(xliff, url);
 
     this._errors = xml.errors;
     ml.visitAll(this, xml.rootNodes, null);
@@ -293,7 +293,7 @@ class XmlToI18n implements ml.Visitor {
   private _errors !: I18nError[];
 
   convert(message: string, url: string) {
-    const xmlIcu = new XmlParser().parse(message, url, true);
+    const xmlIcu = new XmlParser().parse(message, url, {tokenizeExpansionForms: true});
     this._errors = xmlIcu.errors;
 
     const i18nNodes = this._errors.length > 0 || xmlIcu.rootNodes.length == 0 ?

diff --git a/packages/compiler/src/i18n/serializers/xtb.ts b/packages/compiler/src/i18n/serializers/xtb.ts
@@ -88,7 +88,7 @@ class XtbParser implements ml.Visitor {
 
     // We can not parse the ICU messages at this point as some messages might not originate
     // from Angular that could not be lex'd.
-    const xml = new XmlParser().parse(xtb, url, false);
+    const xml = new XmlParser().parse(xtb, url);
 
     this._errors = xml.errors;
     ml.visitAll(this, xml.rootNodes);
@@ -159,7 +159,7 @@ class XmlToI18n implements ml.Visitor {
   private _errors !: I18nError[];
 
   convert(message: string, url: string) {
-    const xmlIcu = new XmlParser().parse(message, url, true);
+    const xmlIcu = new XmlParser().parse(message, url, {tokenizeExpansionForms: true});
     this._errors = xmlIcu.errors;
 
     const i18nNodes = this._errors.length > 0 || xmlIcu.rootNodes.length == 0 ?

diff --git a/packages/compiler/src/i18n/translation_bundle.ts b/packages/compiler/src/i18n/translation_bundle.ts
@@ -83,7 +83,7 @@ class I18nToHtmlVisitor implements i18n.Visitor {
 
     // text to html
     const url = srcMsg.nodes[0].sourceSpan.start.file.url;
-    const html = new HtmlParser().parse(text, url, true);
+    const html = new HtmlParser().parse(text, url, {tokenizeExpansionForms: true});
 
     return {
       nodes: html.rootNodes,

diff --git a/packages/compiler/src/jit_compiler_facade.ts b/packages/compiler/src/jit_compiler_facade.ts
@@ -111,7 +111,7 @@ export class CompilerFacadeImpl implements CompilerFacade {
     // Parse the template and check for errors.
     const template = parseTemplate(
         facade.template, sourceMapUrl,
-        {preserveWhitespaces: facade.preserveWhitespaces || false, interpolationConfig});
+        {preserveWhitespaces: facade.preserveWhitespaces, interpolationConfig});
     if (template.errors !== undefined) {
       const errors = template.errors.map(err => err.toString()).join(', ');
       throw new Error(`Errors during JIT compilation of template for ${facade.name}: ${errors}`);

diff --git a/packages/compiler/src/ml_parser/html_parser.ts b/packages/compiler/src/ml_parser/html_parser.ts
@@ -7,17 +7,15 @@
  */
 
 import {getHtmlTagDefinition} from './html_tags';
-import {DEFAULT_INTERPOLATION_CONFIG, InterpolationConfig} from './interpolation_config';
+import {TokenizeOptions} from './lexer';
 import {ParseTreeResult, Parser} from './parser';
 
 export {ParseTreeResult, TreeError} from './parser';
 
 export class HtmlParser extends Parser {
   constructor() { super(getHtmlTagDefinition); }
 
-  parse(
-      source: string, url: string, parseExpansionForms: boolean = false,
-      interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ParseTreeResult {
-    return super.parse(source, url, parseExpansionForms, interpolationConfig);
+  parse(source: string, url: string, options?: TokenizeOptions): ParseTreeResult {
+    return super.parse(source, url, options);
   }
 }
diff --git a/packages/compiler/src/ml_parser/lexer.ts b/packages/compiler/src/ml_parser/lexer.ts
@@ -36,11 +36,12 @@ export enum TokenType {
 }
 
 export class Token {
-  constructor(public type: TokenType, public parts: string[], public sourceSpan: ParseSourceSpan) {}
+  constructor(
+      public type: TokenType|null, public parts: string[], public sourceSpan: ParseSourceSpan) {}
 }
 
 export class TokenError extends ParseError {
-  constructor(errorMsg: string, public tokenType: TokenType, span: ParseSourceSpan) {
+  constructor(errorMsg: string, public tokenType: TokenType|null, span: ParseSourceSpan) {
     super(span, errorMsg);
   }
 }
@@ -49,14 +50,20 @@ export class TokenizeResult {
   constructor(public tokens: Token[], public errors: TokenError[]) {}
 }
 
+/**
+ * Options that modify how the text is tokenized.
+ */
+export interface TokenizeOptions {
+  /** Whether to tokenize ICU messages (considered as text nodes when false). */
+  tokenizeExpansionForms?: boolean;
+  /** How to tokenize interpolation markers. */
+  interpolationConfig?: InterpolationConfig;
+}
+
 export function tokenize(
     source: string, url: string, getTagDefinition: (tagName: string) => TagDefinition,
-    tokenizeExpansionForms: boolean = false,
-    interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): TokenizeResult {
-  return new _Tokenizer(
-             new ParseSourceFile(source, url), getTagDefinition, tokenizeExpansionForms,
-             interpolationConfig)
-      .tokenize();
+    options: TokenizeOptions = {}): TokenizeResult {
+  return new _Tokenizer(new ParseSourceFile(source, url), getTagDefinition, options).tokenize();
 }
 
 const _CR_OR_CRLF_REGEXP = /\r\n?/g;
@@ -78,16 +85,15 @@ class _ControlFlowError {
 class _Tokenizer {
   private _input: string;
   private _length: number;
-  // Note: this is always lowercase!
+  private _tokenizeIcu: boolean;
+  private _interpolationConfig: InterpolationConfig;
   private _peek: number = -1;
   private _nextPeek: number = -1;
   private _index: number = -1;
   private _line: number = 0;
   private _column: number = -1;
-  // TODO(issue/24571): remove '!'.
-  private _currentTokenStart !: ParseLocation;
-  // TODO(issue/24571): remove '!'.
-  private _currentTokenType !: TokenType;
+  private _currentTokenStart: ParseLocation|null = null;
+  private _currentTokenType: TokenType|null = null;
   private _expansionCaseStack: TokenType[] = [];
   private _inInterpolation: boolean = false;
 
@@ -102,8 +108,9 @@ class _Tokenizer {
    */
   constructor(
       private _file: ParseSourceFile, private _getTagDefinition: (tagName: string) => TagDefinition,
-      private _tokenizeIcu: boolean,
-      private _interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG) {
+      options: TokenizeOptions) {
+    this._tokenizeIcu = options.tokenizeExpansionForms || false;
+    this._interpolationConfig = options.interpolationConfig || DEFAULT_INTERPOLATION_CONFIG;
     this._input = _file.content;
     this._length = _file.content.length;
     this._advance();
@@ -197,11 +204,21 @@ class _Tokenizer {
   }
 
   private _endToken(parts: string[], end: ParseLocation = this._getLocation()): Token {
+    if (this._currentTokenStart === null) {
+      throw new TokenError(
+          'Programming error - attempted to end a token when there was no start to the token',
+          this._currentTokenType, this._getSpan(end, end));
+    }
+    if (this._currentTokenType === null) {
+      throw new TokenError(
+          'Programming error - attempted to end a token which has no token type', null,
+          this._getSpan(this._currentTokenStart, end));
+    }
     const token =
         new Token(this._currentTokenType, parts, new ParseSourceSpan(this._currentTokenStart, end));
     this.tokens.push(token);
-    this._currentTokenStart = null !;
-    this._currentTokenType = null !;
+    this._currentTokenStart = null;
+    this._currentTokenType = null;
     return token;
   }
 

diff --git a/packages/compiler/src/ml_parser/parser.ts b/packages/compiler/src/ml_parser/parser.ts
@@ -9,7 +9,6 @@
 import {ParseError, ParseSourceSpan} from '../parse_util';
 
 import * as html from './ast';
-import {DEFAULT_INTERPOLATION_CONFIG, InterpolationConfig} from './interpolation_config';
 import * as lex from './lexer';
 import {TagDefinition, getNsPrefix, isNgContainer, mergeNsAndName} from './tags';
 
@@ -30,11 +29,8 @@ export class ParseTreeResult {
 export class Parser {
   constructor(public getTagDefinition: (tagName: string) => TagDefinition) {}
 
-  parse(
-      source: string, url: string, parseExpansionForms: boolean = false,
-      interpolationConfig: InterpolationConfig = DEFAULT_INTERPOLATION_CONFIG): ParseTreeResult {
-    const tokensAndErrors =
-        lex.tokenize(source, url, this.getTagDefinition, parseExpansionForms, interpolationConfig);
+  parse(source: string, url: string, options?: lex.TokenizeOptions): ParseTreeResult {
+    const tokensAndErrors = lex.tokenize(source, url, this.getTagDefinition, options);
 
     const treeAndErrors = new _TreeBuilder(tokensAndErrors.tokens, this.getTagDefinition).build();
 

diff --git a/packages/compiler/src/ml_parser/xml_parser.ts b/packages/compiler/src/ml_parser/xml_parser.ts
@@ -6,6 +6,7 @@
  * found in the LICENSE file at https://angular.io/license
  */
 
+import {TokenizeOptions} from './lexer';
 import {ParseTreeResult, Parser} from './parser';
 import {getXmlTagDefinition} from './xml_tags';
 
@@ -14,7 +15,7 @@ export {ParseTreeResult, TreeError} from './parser';
 export class XmlParser extends Parser {
   constructor() { super(getXmlTagDefinition); }
 
-  parse(source: string, url: string, parseExpansionForms: boolean = false): ParseTreeResult {
-    return super.parse(source, url, parseExpansionForms);
+  parse(source: string, url: string, options?: TokenizeOptions): ParseTreeResult {
+    return super.parse(source, url, options);
   }
 }
diff --git a/packages/compiler/src/render3/view/template.ts b/packages/compiler/src/render3/view/template.ts
@@ -1498,20 +1498,35 @@ function interpolate(args: o.Expression[]): o.Expression {
   return o.importExpr(R3.interpolationV).callFn([o.literalArr(args)]);
 }
 
+/**
+ * Options that can be used to modify how a template is parsed by `parseTemplate()`.
+ */
+export interface ParseTemplateOptions {
+  /**
+   * Include whitespace nodes in the parsed output.
+   */
+  preserveWhitespaces?: boolean;
+  /**
+   * How to parse interpolation markers.
+   */
+  interpolationConfig?: InterpolationConfig;
+}
+
 /**
  * Parse a template into render3 `Node`s and additional metadata, with no other dependencies.
  *
  * @param template text of the template to parse
  * @param templateUrl URL to use for source mapping of the parsed template
+ * @param options options to modify how the template is parsed
  */
 export function parseTemplate(
     template: string, templateUrl: string,
-    options: {preserveWhitespaces?: boolean, interpolationConfig?: InterpolationConfig} = {}):
-    {errors?: ParseError[], nodes: t.Node[]} {
+    options: ParseTemplateOptions = {}): {errors?: ParseError[], nodes: t.Node[]} {
   const {interpolationConfig, preserveWhitespaces} = options;
   const bindingParser = makeBindingParser(interpolationConfig);
   const htmlParser = new HtmlParser();
-  const parseResult = htmlParser.parse(template, templateUrl, true, interpolationConfig);
+  const parseResult =
+      htmlParser.parse(template, templateUrl, {...options, tokenizeExpansionForms: true});
 
   if (parseResult.errors && parseResult.errors.length > 0) {
     return {errors: parseResult.errors, nodes: []};

diff --git a/packages/compiler/src/template_parser/template_parser.ts b/packages/compiler/src/template_parser/template_parser.ts
@@ -114,8 +114,10 @@ export class TemplateParser {
       directives: CompileDirectiveSummary[], pipes: CompilePipeSummary[], schemas: SchemaMetadata[],
       templateUrl: string, preserveWhitespaces: boolean): TemplateParseResult {
     let htmlParseResult = typeof template === 'string' ?
-        this._htmlParser !.parse(
-            template, templateUrl, true, this.getInterpolationConfig(component)) :
+        this._htmlParser !.parse(template, templateUrl, {
+          tokenizeExpansionForms: true,
+          interpolationConfig: this.getInterpolationConfig(component)
+        }) :
         template;
 
     if (!preserveWhitespaces) {

diff --git a/packages/compiler/test/i18n/extractor_merger_spec.ts b/packages/compiler/test/i18n/extractor_merger_spec.ts
@@ -501,7 +501,7 @@ import {serializeNodes as serializeHtmlNodes} from '../ml_parser/util/util';
 
 function parseHtml(html: string): html.Node[] {
   const htmlParser = new HtmlParser();
-  const parseResult = htmlParser.parse(html, 'extractor spec', true);
+  const parseResult = htmlParser.parse(html, 'extractor spec', {tokenizeExpansionForms: true});
   if (parseResult.errors.length > 1) {
     throw new Error(`unexpected parse errors: ${parseResult.errors.join('\n')}`);
   }

diff --git a/packages/compiler/test/i18n/i18n_parser_spec.ts b/packages/compiler/test/i18n/i18n_parser_spec.ts
@@ -330,7 +330,7 @@ export function _extractMessages(
     html: string, implicitTags: string[] = [],
     implicitAttrs: {[k: string]: string[]} = {}): Message[] {
   const htmlParser = new HtmlParser();
-  const parseResult = htmlParser.parse(html, 'extractor spec', true);
+  const parseResult = htmlParser.parse(html, 'extractor spec', {tokenizeExpansionForms: true});
   if (parseResult.errors.length > 1) {
     throw Error(`unexpected parse errors: ${parseResult.errors.join('\n')}`);
   }