Implement support for "$XZY" style environment variables

pgonzal · pgonzal · commit 9c8064cf7e1b · 2018-06-21T17:22:49.000-07:00
diff --git a/libraries/rushell/src/Tokenizer.ts b/libraries/rushell/src/Tokenizer.ts
@@ -46,12 +46,18 @@ export class Token {
   }
 }
 
-const wordCharacterOrBackslashRegExp: RegExp = /[a-z0-9_\\]/i;
+const textCharacterRegExp: RegExp = /[a-z0-9_\\]/i;
+const startVariableCharacterRegExp: RegExp = /[a-z_]/i;
+const variableCharacterRegExp: RegExp = /[a-z0-9_]/i;
 
 export class Tokenizer {
   public readonly input: TextRange;
   private _currentIndex: number;
 
+  private static _isSpace(c: string | undefined): boolean {
+    return c === ' ' || c === '\t';
+  }
+
   constructor(input: TextRange | string) {
     if (typeof(input) === 'string') {
       this.input = TextRange.fromString(input);
@@ -77,7 +83,7 @@ export class Tokenizer {
     }
 
     // Is it a sequence of whitespace?
-    if (/[ \t]/.test(firstChar)) {
+    if (Tokenizer._isSpace(firstChar)) {
       this._get();
 
       while (Tokenizer._isSpace(this._peek())) {
@@ -100,30 +106,45 @@ export class Tokenizer {
     }
 
     // Is it a text token?
-    if (wordCharacterOrBackslashRegExp.test(firstChar)) {
+    if (textCharacterRegExp.test(firstChar)) {
       let text: string = '';
       let c: string | undefined = firstChar;
       do {
         if (c === '\\') {
           this._get(); // discard the backslash
           if (this._peek() === undefined) {
             throw new ParseError('Backslash encountered at end of stream',
-              input.getNewRange(this._currentIndex, this._currentIndex+1));
+              input.getNewRange(this._currentIndex, this._currentIndex + 1));
           }
           text += this._get();
         } else {
           text += this._get();
         }
 
         c = this._peek();
-        if (c == undefined) {
-          break;
-        }
-      } while (wordCharacterOrBackslashRegExp.test(c));
+      } while (c && textCharacterRegExp.test(c));
 
       return new Token(TokenKind.Text, input.getNewRange(startIndex, this._currentIndex), text);
     }
 
+    // Is it a dollar variable?  The valid environment variable names are [A-Z_][A-Z0-9_]*
+    if (firstChar === '$') {
+      this._get();
+
+      let name: string = this._get() || '';
+      if (!startVariableCharacterRegExp.test(name)) {
+        throw new ParseError('The "$" symbol must be followed by a letter or underscore',
+          input.getNewRange(startIndex, this._currentIndex));
+      }
+
+      let c: string | undefined = this._peek();
+      while (c && variableCharacterRegExp.test(c)) {
+        name += this._get();
+        c = this._peek();
+      }
+      return new Token(TokenKind.DollarVariable, input.getNewRange(startIndex, this._currentIndex), name);
+    }
+
     // Is it the "&&" token?
     if (firstChar === '&') {
       if (this._peek2() === '&') {
@@ -163,13 +184,9 @@ export class Tokenizer {
   }
 
   private _peek2(): string | undefined {
-    if (this._currentIndex+1 >= this.input.end) {
+    if (this._currentIndex + 1 >= this.input.end) {
       return undefined;
     }
-    return this.input.buffer[this._currentIndex+1];
-  }
-
-  private static _isSpace(c: string | undefined): boolean {
-    return c === ' ' || c === '\t';
+    return this.input.buffer[this._currentIndex + 1];
   }
 }
diff --git a/libraries/rushell/src/test/Tokenizer.test.ts b/libraries/rushell/src/test/Tokenizer.test.ts
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
 
-import { Tokenizer, TokenKind } from '../Tokenizer';
+import { Tokenizer, TokenKind, Token } from '../Tokenizer';
 
 function escape(s: string): string {
   return s.replace(/\n/g, '[n]')
@@ -10,8 +10,13 @@ function escape(s: string): string {
     .replace(/\\/g, '[b]');
 }
 
+function tokenize(input: string): Token[] {
+  const tokenizer: Tokenizer = new Tokenizer(input);
+  return tokenizer.getTokens();
+}
+
 function matchSnapshot(input: string): void {
-  const tokenizer = new Tokenizer(input);
+  const tokenizer: Tokenizer = new Tokenizer(input);
   expect({
     input: escape(tokenizer.input.toString()),
     tokens: tokenizer.getTokens().map(x => [TokenKind[x.kind], escape(x.toString())])
@@ -28,7 +33,7 @@ test('01: white space tokens', () => {
 });
 
 test('02: text with escapes', () => {
-  matchSnapshot(' ab+56\\>qrst$(abc\\))');
+  matchSnapshot(' ab+56\\>qrst(abc\\))');
 });
 
 test('03: The && operator', () => {
@@ -37,3 +42,10 @@ test('03: The && operator', () => {
   matchSnapshot('&&');
   matchSnapshot('&');
 });
+
+test('04: dollar variables', () => {
+  matchSnapshot('$abc123.456');
+  matchSnapshot('$ab$_90');
+  expect(() => tokenize('$')).toThrowError();
+  expect(() => tokenize('${abc}')).toThrowError();
+});
diff --git a/libraries/rushell/src/test/__snapshots__/Tokenizer.test.ts.snap b/libraries/rushell/src/test/__snapshots__/Tokenizer.test.ts.snap
@@ -77,7 +77,7 @@ Object {
 
 exports[`02: text with escapes 1`] = `
 Object {
-  "input": " ab+56[b]>qrst$(abc[b]))",
+  "input": " ab+56[b]>qrst(abc[b]))",
   "tokens": Array [
     Array [
       "Spaces",
@@ -95,10 +95,6 @@ Object {
       "Text",
       "56>qrst",
     ],
-    Array [
-      "Other",
-      "$",
-    ],
     Array [
       "Other",
       "(",
@@ -186,3 +182,39 @@ Object {
   ],
 }
 `;
+
+exports[`04: dollar variables 1`] = `
+Object {
+  "input": "$abc123.456",
+  "tokens": Array [
+    Array [
+      "DollarVariable",
+      "abc123",
+    ],
+    Array [
+      "Other",
+      ".",
+    ],
+    Array [
+      "Text",
+      "456",
+    ],
+  ],
+}
+`;
+
+exports[`04: dollar variables 2`] = `
+Object {
+  "input": "$ab$_90",
+  "tokens": Array [
+    Array [
+      "DollarVariable",
+      "ab",
+    ],
+    Array [
+      "DollarVariable",
+      "_90",
+    ],
+  ],
+}
+`;
