Skip to content

Commit 2e9eb2d

Browse files
committed
py: Fix lexer so it doesn't allow ! and ..
1 parent 175cecf commit 2e9eb2d

1 file changed

Lines changed: 21 additions & 3 deletions

File tree

py/lexer.c

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,8 +218,7 @@ STATIC const char *tok_enc =
218218
"%e=" // % %=
219219
"^e=" // ^ ^=
220220
"=e=" // = ==
221-
"!E=" // !=
222-
".c.E."; // . ...
221+
"!E="; // !=
223222

224223
// TODO static assert that number of tokens is less than 256 so we can safely make this table with byte sized entries
225224
STATIC const uint8_t tok_enc_kind[] = {
@@ -240,7 +239,6 @@ STATIC const uint8_t tok_enc_kind[] = {
240239
MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL,
241240
MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL,
242241
MP_TOKEN_OP_NOT_EQUAL,
243-
MP_TOKEN_DEL_PERIOD, MP_TOKEN_ELLIPSIS,
244242
};
245243

246244
// must have the same order as enum in lexer.h
@@ -560,6 +558,23 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
560558
}
561559
}
562560

561+
} else if (is_char(lex, '.')) {
562+
// special handling for . and ... operators, because .. is not a valid operator
563+
564+
// get first char
565+
vstr_add_char(&lex->vstr, '.');
566+
next_char(lex);
567+
568+
if (is_char_and(lex, '.', '.')) {
569+
vstr_add_char(&lex->vstr, '.');
570+
vstr_add_char(&lex->vstr, '.');
571+
next_char(lex);
572+
next_char(lex);
573+
tok->kind = MP_TOKEN_ELLIPSIS;
574+
} else {
575+
tok->kind = MP_TOKEN_DEL_PERIOD;
576+
}
577+
563578
} else {
564579
// search for encoded delimiter or operator
565580

@@ -605,6 +620,7 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
605620
tok_enc_index = t_index;
606621
} else {
607622
tok->kind = MP_TOKEN_INVALID;
623+
goto tok_enc_no_match;
608624
}
609625
break;
610626
}
@@ -627,6 +643,8 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
627643
// set token kind
628644
tok->kind = tok_enc_kind[tok_enc_index];
629645

646+
tok_enc_no_match:
647+
630648
// compute bracket level for implicit line joining
631649
if (tok->kind == MP_TOKEN_DEL_PAREN_OPEN || tok->kind == MP_TOKEN_DEL_BRACKET_OPEN || tok->kind == MP_TOKEN_DEL_BRACE_OPEN) {
632650
lex->nested_bracket_level += 1;

0 commit comments

Comments
 (0)