Implement new "text" token type

pgonzal · pgonzal · commit e825500a0a36 · 2018-06-21T17:22:48.000-07:00
diff --git a/libraries/rushell/src/Tokenizer.ts b/libraries/rushell/src/Tokenizer.ts
@@ -2,6 +2,7 @@
 // See LICENSE in the project root for license information.
 
 import { TextRange } from './TextRange';
+import { ParseError } from './ParseError';
 
 export enum TokenKind {
   // One or more spaces/tabs
@@ -10,24 +11,32 @@ export enum TokenKind {
   NewLine,
   // An unrecognized character
   Other,
+  // A sequence of characters that doesn't contain any symbols with special meaning
+  // Characters can be escaped, in which case the Token.text may differ from the
+  // Token.range.toString()
+  Text,
   // The end of the input string
   EndOfInput
 }
 
 export class Token {
   public readonly kind: TokenKind;
   public readonly range: TextRange;
+  public readonly text: string;
 
-  public constructor(kind: TokenKind, range: TextRange) {
+  public constructor(kind: TokenKind, range: TextRange, text?: string) {
     this.kind = kind;
     this.range = range;
+    this.text = text === undefined ? this.range.toString() : text;
   }
 
   public toString(): string {
-    return this.range.toString();
+    return this.text;
   }
 }
 
+const wordCharacterOrBackslashRegExp: RegExp = /[a-z0-9_\\]/i;
+
 export class Tokenizer {
   public readonly input: TextRange;
   private _currentIndex: number;
@@ -49,7 +58,7 @@ export class Tokenizer {
     const input: TextRange = this.input;
 
     const startIndex: number = this._currentIndex;
-    let c: string | undefined = this._get();
+    let c: string | undefined = this._peek();
 
     // Reached end of input yet?
     if (c === undefined) {
@@ -58,6 +67,7 @@ export class Tokenizer {
 
     // Is it a sequence of whitespace?
     if (/[ \t]/.test(c)) {
+      this._get();
 
       while (Tokenizer._isSpace(this._peek())) {
         this._get();
@@ -68,15 +78,39 @@ export class Tokenizer {
 
     // Is it a newline?
     if (c === '\r') {
+      this._get();
       if (this._peek() === '\n') {
         this._get();
       }
       return new Token(TokenKind.NewLine, input.getNewRange(startIndex, this._currentIndex));
     } else if (c === '\n') {
+      this._get();
       return new Token(TokenKind.NewLine, input.getNewRange(startIndex, this._currentIndex));
     }
 
+    // Is it a text token?
+    if (wordCharacterOrBackslashRegExp.test(c)) {
+      let text: string = '';
+      while (wordCharacterOrBackslashRegExp.test(c)) {
+        if (c === '\\') {
+          this._get(); // discard the backslash
+          if (this._peek() === undefined) {
+            throw new ParseError('Backslash encountered at end of stream',
+              input.getNewRange(this._currentIndex, this._currentIndex+1));
+          }
+          text += this._get();
+        } else {
+          text += this._get();
+        }
+
+        c = this._peek();
+      }
+
+      return new Token(TokenKind.Text, input.getNewRange(startIndex, this._currentIndex), text);
+    }
+
     // Otherwise treat it as an "other" character
+    this._get();
     return new Token(TokenKind.Other, input.getNewRange(startIndex, this._currentIndex));
   }
 
diff --git a/libraries/rushell/src/test/__snapshots__/Tokenizer.test.ts.snap b/libraries/rushell/src/test/__snapshots__/Tokenizer.test.ts.snap
@@ -21,23 +21,15 @@ Object {
 
 exports[`white space tokens 1`] = `
 Object {
-  "input": "  abc   [r][n]def  [n]  ghi[n][r]  ",
+  "input": " [t] abc   [r][n]def  [n]  ghi[n][r]  ",
   "tokens": Array [
     Array [
       "Spaces",
-      "  ",
-    ],
-    Array [
-      "Other",
-      "a",
+      " [t] ",
     ],
     Array [
-      "Other",
-      "b",
-    ],
-    Array [
-      "Other",
-      "c",
+      "Text",
+      "abc",
     ],
     Array [
       "Spaces",
@@ -48,16 +40,8 @@ Object {
       "[r][n]",
     ],
     Array [
-      "Other",
-      "d",
-    ],
-    Array [
-      "Other",
-      "e",
-    ],
-    Array [
-      "Other",
-      "f",
+      "Text",
+      "def",
     ],
     Array [
       "Spaces",
@@ -72,16 +56,8 @@ Object {
       "  ",
     ],
     Array [
-      "Other",
-      "g",
-    ],
-    Array [
-      "Other",
-      "h",
-    ],
-    Array [
-      "Other",
-      "i",
+      "Text",
+      "ghi",
     ],
     Array [
       "NewLine",
