Skip to content

Commit 349839f

Browse files
vsavkinRobert Messerle
vsavkin
authored and
Robert Messerle
committed
feat(html_lexer): support special forms used by i18n { exp, plural, =0 {} }
1 parent 9d10c09 commit 349839f

File tree

2 files changed

+202
-11
lines changed

2 files changed

+202
-11
lines changed

‎modules/angular2/src/compiler/html_lexer.ts‎

Lines changed: 124 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@ export enum HtmlTokenType {
2525
ATTR_NAME,
2626
ATTR_VALUE,
2727
DOC_TYPE,
28+
EXPANSION_FORM_START,
29+
EXPANSION_CASE_VALUE,
30+
EXPANSION_CASE_EXP_START,
31+
EXPANSION_CASE_EXP_END,
32+
EXPANSION_FORM_END,
2833
EOF
2934
}
3035

@@ -43,8 +48,10 @@ export class HtmlTokenizeResult {
4348
constructor(public tokens: HtmlToken[], public errors: HtmlTokenError[]) {}
4449
}
4550

46-
export function tokenizeHtml(sourceContent: string, sourceUrl: string): HtmlTokenizeResult {
47-
return new _HtmlTokenizer(new ParseSourceFile(sourceContent, sourceUrl)).tokenize();
51+
export function tokenizeHtml(sourceContent: string, sourceUrl: string,
52+
tokenizeExpansionForms: boolean = false): HtmlTokenizeResult {
53+
return new _HtmlTokenizer(new ParseSourceFile(sourceContent, sourceUrl), tokenizeExpansionForms)
54+
.tokenize();
4855
}
4956

5057
const $EOF = 0;
@@ -75,6 +82,9 @@ const $GT = 62;
7582
const $QUESTION = 63;
7683
const $LBRACKET = 91;
7784
const $RBRACKET = 93;
85+
const $LBRACE = 123;
86+
const $RBRACE = 125;
87+
const $COMMA = 44;
7888
const $A = 65;
7989
const $F = 70;
8090
const $X = 88;
@@ -108,16 +118,20 @@ class _HtmlTokenizer {
108118
private length: number;
109119
// Note: this is always lowercase!
110120
private peek: number = -1;
121+
private nextPeek: number = -1;
111122
private index: number = -1;
112123
private line: number = 0;
113124
private column: number = -1;
114125
private currentTokenStart: ParseLocation;
115126
private currentTokenType: HtmlTokenType;
116127

128+
private inExpansionCase: boolean = false;
129+
private inExpansionForm: boolean = false;
130+
117131
tokens: HtmlToken[] = [];
118132
errors: HtmlTokenError[] = [];
119133

120-
constructor(private file: ParseSourceFile) {
134+
constructor(private file: ParseSourceFile, private tokenizeExpansionForms: boolean) {
121135
this.input = file.content;
122136
this.length = file.content.length;
123137
this._advance();
@@ -149,6 +163,18 @@ class _HtmlTokenizer {
149163
} else {
150164
this._consumeTagOpen(start);
151165
}
166+
} else if (isSpecialFormStart(this.peek, this.nextPeek) && this.tokenizeExpansionForms) {
167+
this._consumeExpansionFormStart();
168+
169+
} else if (this.peek === $EQ && this.tokenizeExpansionForms) {
170+
this._consumeExpansionCaseStart();
171+
172+
} else if (this.peek === $RBRACE && this.inExpansionCase && this.tokenizeExpansionForms) {
173+
this._consumeExpansionCaseEnd();
174+
175+
} else if (this.peek === $RBRACE && !this.inExpansionCase && this.tokenizeExpansionForms) {
176+
this._consumeExpansionFormEnd();
177+
152178
} else {
153179
this._consumeText();
154180
}
@@ -218,6 +244,8 @@ class _HtmlTokenizer {
218244
}
219245
this.index++;
220246
this.peek = this.index >= this.length ? $EOF : StringWrapper.charCodeAt(this.input, this.index);
247+
this.nextPeek =
248+
this.index + 1 >= this.length ? $EOF : StringWrapper.charCodeAt(this.input, this.index + 1);
221249
}
222250

223251
private _attemptCharCode(charCode: number): boolean {
@@ -506,20 +534,109 @@ class _HtmlTokenizer {
506534
this._endToken(prefixAndName);
507535
}
508536

537+
private _consumeExpansionFormStart() {
538+
this._beginToken(HtmlTokenType.EXPANSION_FORM_START, this._getLocation());
539+
this._requireCharCode($LBRACE);
540+
this._endToken([]);
541+
542+
this._beginToken(HtmlTokenType.RAW_TEXT, this._getLocation());
543+
let condition = this._readUntil($COMMA);
544+
this._endToken([condition], this._getLocation());
545+
this._requireCharCode($COMMA);
546+
this._attemptCharCodeUntilFn(isNotWhitespace);
547+
548+
this._beginToken(HtmlTokenType.RAW_TEXT, this._getLocation());
549+
let type = this._readUntil($COMMA);
550+
this._endToken([type], this._getLocation());
551+
this._requireCharCode($COMMA);
552+
this._attemptCharCodeUntilFn(isNotWhitespace);
553+
554+
this.inExpansionForm = true;
555+
}
556+
557+
private _consumeExpansionCaseStart() {
558+
this._requireCharCode($EQ);
559+
560+
this._beginToken(HtmlTokenType.EXPANSION_CASE_VALUE, this._getLocation());
561+
let value = this._readUntil($LBRACE).trim();
562+
this._endToken([value], this._getLocation());
563+
this._attemptCharCodeUntilFn(isNotWhitespace);
564+
565+
this._beginToken(HtmlTokenType.EXPANSION_CASE_EXP_START, this._getLocation());
566+
this._requireCharCode($LBRACE);
567+
this._endToken([], this._getLocation());
568+
this._attemptCharCodeUntilFn(isNotWhitespace);
569+
570+
this.inExpansionCase = true;
571+
}
572+
573+
private _consumeExpansionCaseEnd() {
574+
this._beginToken(HtmlTokenType.EXPANSION_CASE_EXP_END, this._getLocation());
575+
this._requireCharCode($RBRACE);
576+
this._endToken([], this._getLocation());
577+
this._attemptCharCodeUntilFn(isNotWhitespace);
578+
579+
this.inExpansionCase = false;
580+
}
581+
582+
private _consumeExpansionFormEnd() {
583+
this._beginToken(HtmlTokenType.EXPANSION_FORM_END, this._getLocation());
584+
this._requireCharCode($RBRACE);
585+
this._endToken([]);
586+
587+
this.inExpansionForm = false;
588+
}
589+
509590
private _consumeText() {
510591
var start = this._getLocation();
511592
this._beginToken(HtmlTokenType.TEXT, start);
512-
var parts = [this._readChar(true)];
513-
while (!isTextEnd(this.peek)) {
593+
594+
var parts = [];
595+
let interpolation = false;
596+
597+
if (this.peek === $LBRACE && this.nextPeek === $LBRACE) {
598+
parts.push(this._readChar(true));
599+
parts.push(this._readChar(true));
600+
interpolation = true;
601+
} else {
514602
parts.push(this._readChar(true));
515603
}
604+
605+
while (!this.isTextEnd(interpolation)) {
606+
if (this.peek === $LBRACE && this.nextPeek === $LBRACE) {
607+
parts.push(this._readChar(true));
608+
parts.push(this._readChar(true));
609+
interpolation = true;
610+
} else if (this.peek === $RBRACE && this.nextPeek === $RBRACE && interpolation) {
611+
parts.push(this._readChar(true));
612+
parts.push(this._readChar(true));
613+
interpolation = false;
614+
} else {
615+
parts.push(this._readChar(true));
616+
}
617+
}
516618
this._endToken([this._processCarriageReturns(parts.join(''))]);
517619
}
518620

621+
private isTextEnd(interpolation: boolean): boolean {
622+
if (this.peek === $LT || this.peek === $EOF) return true;
623+
if (this.tokenizeExpansionForms) {
624+
if (isSpecialFormStart(this.peek, this.nextPeek)) return true;
625+
if (this.peek === $RBRACE && !interpolation && this.inExpansionForm) return true;
626+
}
627+
return false;
628+
}
629+
519630
private _savePosition(): number[] {
520631
return [this.peek, this.index, this.column, this.line, this.tokens.length];
521632
}
522633

634+
private _readUntil(char: number): string {
635+
let start = this.index;
636+
this._attemptUntilChar(char);
637+
return this.input.substring(start, this.index);
638+
}
639+
523640
private _restorePosition(position: number[]): void {
524641
this.peek = position[0];
525642
this.index = position[1];
@@ -558,8 +675,8 @@ function isNamedEntityEnd(code: number): boolean {
558675
return code == $SEMICOLON || code == $EOF || !isAsciiLetter(code);
559676
}
560677

561-
function isTextEnd(code: number): boolean {
562-
return code === $LT || code === $EOF;
678+
function isSpecialFormStart(peek: number, nextPeek: number): boolean {
679+
return peek === $LBRACE && nextPeek != $LBRACE;
563680
}
564681

565682
function isAsciiLetter(code: number): boolean {

‎modules/angular2/test/compiler/html_lexer_spec.ts‎

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -576,6 +576,78 @@ export function main() {
576576

577577
});
578578

579+
describe("expansion forms", () => {
580+
it("should parse an expansion form", () => {
581+
expect(tokenizeAndHumanizeParts('{one.two, three, =4 {four} =5 {five} }', true))
582+
.toEqual([
583+
[HtmlTokenType.EXPANSION_FORM_START],
584+
[HtmlTokenType.RAW_TEXT, 'one.two'],
585+
[HtmlTokenType.RAW_TEXT, 'three'],
586+
[HtmlTokenType.EXPANSION_CASE_VALUE, '4'],
587+
[HtmlTokenType.EXPANSION_CASE_EXP_START],
588+
[HtmlTokenType.TEXT, 'four'],
589+
[HtmlTokenType.EXPANSION_CASE_EXP_END],
590+
[HtmlTokenType.EXPANSION_CASE_VALUE, '5'],
591+
[HtmlTokenType.EXPANSION_CASE_EXP_START],
592+
[HtmlTokenType.TEXT, 'five'],
593+
[HtmlTokenType.EXPANSION_CASE_EXP_END],
594+
[HtmlTokenType.EXPANSION_FORM_END],
595+
[HtmlTokenType.EOF]
596+
]);
597+
});
598+
599+
it("should parse an expansion form with text elements surrounding it", () => {
600+
expect(tokenizeAndHumanizeParts('before{one.two, three, =4 {four}}after', true))
601+
.toEqual([
602+
[HtmlTokenType.TEXT, "before"],
603+
[HtmlTokenType.EXPANSION_FORM_START],
604+
[HtmlTokenType.RAW_TEXT, 'one.two'],
605+
[HtmlTokenType.RAW_TEXT, 'three'],
606+
[HtmlTokenType.EXPANSION_CASE_VALUE, '4'],
607+
[HtmlTokenType.EXPANSION_CASE_EXP_START],
608+
[HtmlTokenType.TEXT, 'four'],
609+
[HtmlTokenType.EXPANSION_CASE_EXP_END],
610+
[HtmlTokenType.EXPANSION_FORM_END],
611+
[HtmlTokenType.TEXT, "after"],
612+
[HtmlTokenType.EOF]
613+
]);
614+
});
615+
616+
it("should parse an expansion forms with elements in it", () => {
617+
expect(tokenizeAndHumanizeParts('{one.two, three, =4 {four <b>a</b>}}', true))
618+
.toEqual([
619+
[HtmlTokenType.EXPANSION_FORM_START],
620+
[HtmlTokenType.RAW_TEXT, 'one.two'],
621+
[HtmlTokenType.RAW_TEXT, 'three'],
622+
[HtmlTokenType.EXPANSION_CASE_VALUE, '4'],
623+
[HtmlTokenType.EXPANSION_CASE_EXP_START],
624+
[HtmlTokenType.TEXT, 'four '],
625+
[HtmlTokenType.TAG_OPEN_START, null, 'b'],
626+
[HtmlTokenType.TAG_OPEN_END],
627+
[HtmlTokenType.TEXT, 'a'],
628+
[HtmlTokenType.TAG_CLOSE, null, 'b'],
629+
[HtmlTokenType.EXPANSION_CASE_EXP_END],
630+
[HtmlTokenType.EXPANSION_FORM_END],
631+
[HtmlTokenType.EOF]
632+
]);
633+
});
634+
635+
it("should parse an expansion forms with interpolation in it", () => {
636+
expect(tokenizeAndHumanizeParts('{one.two, three, =4 {four {{a}}}}', true))
637+
.toEqual([
638+
[HtmlTokenType.EXPANSION_FORM_START],
639+
[HtmlTokenType.RAW_TEXT, 'one.two'],
640+
[HtmlTokenType.RAW_TEXT, 'three'],
641+
[HtmlTokenType.EXPANSION_CASE_VALUE, '4'],
642+
[HtmlTokenType.EXPANSION_CASE_EXP_START],
643+
[HtmlTokenType.TEXT, 'four {{a}}'],
644+
[HtmlTokenType.EXPANSION_CASE_EXP_END],
645+
[HtmlTokenType.EXPANSION_FORM_END],
646+
[HtmlTokenType.EOF]
647+
]);
648+
});
649+
});
650+
579651
describe('errors', () => {
580652
it('should include 2 lines of context in message', () => {
581653
let src = "111\n222\n333\nE\n444\n555\n666\n";
@@ -604,17 +676,19 @@ export function main() {
604676
});
605677
}
606678

607-
function tokenizeWithoutErrors(input: string): HtmlToken[] {
608-
var tokenizeResult = tokenizeHtml(input, 'someUrl');
679+
function tokenizeWithoutErrors(input: string,
680+
tokenizeExpansionForms: boolean = false): HtmlToken[] {
681+
var tokenizeResult = tokenizeHtml(input, 'someUrl', tokenizeExpansionForms);
609682
if (tokenizeResult.errors.length > 0) {
610683
var errorString = tokenizeResult.errors.join('\n');
611684
throw new BaseException(`Unexpected parse errors:\n${errorString}`);
612685
}
613686
return tokenizeResult.tokens;
614687
}
615688

616-
function tokenizeAndHumanizeParts(input: string): any[] {
617-
return tokenizeWithoutErrors(input).map(token => [<any>token.type].concat(token.parts));
689+
function tokenizeAndHumanizeParts(input: string, tokenizeExpansionForms: boolean = false): any[] {
690+
return tokenizeWithoutErrors(input, tokenizeExpansionForms)
691+
.map(token => [<any>token.type].concat(token.parts));
618692
}
619693

620694
function tokenizeAndHumanizeSourceSpans(input: string): any[] {

0 commit comments

Comments
 (0)