Skip to content

Commit ae43679

Browse files
dpgeorgedpgeorge
committed
py/lexer: Use strcmp to make keyword searching more efficient.
Since the table of keywords is sorted, we can use strcmp to do the search and stop part way through the search if the comparison is less-than. Because all tokens that are names are subject to this search, this optimisation will improve the overall speed of the lexer when processing a script. The change also decreases code size by a little bit because we now use strcmp instead of the custom str_strn_equal function.
1 parent a68c754 commit ae43679

2 files changed

Lines changed: 20 additions & 28 deletions

File tree

‎py/lexer.c‎

Lines changed: 11 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
*/
2626

2727
#include <stdio.h>
28+
#include <string.h>
2829
#include <assert.h>
2930

3031
#include "py/mpstate.h"
@@ -39,19 +40,6 @@
3940
// TODO seems that CPython allows NULL byte in the input stream
4041
// don't know if that's intentional or not, but we don't allow it
4142

42-
// TODO replace with a call to a standard function
43-
STATIC bool str_strn_equal(const char *str, const char *strn, mp_uint_t len) {
44-
mp_uint_t i = 0;
45-
46-
while (i < len && *str == *strn) {
47-
++i;
48-
++str;
49-
++strn;
50-
}
51-
52-
return i == len && *str == 0;
53-
}
54-
5543
#define MP_LEXER_EOF ((unichar)MP_READER_EOF)
5644
#define CUR_CHAR(lex) ((lex)->chr0)
5745

@@ -225,10 +213,12 @@ STATIC const uint8_t tok_enc_kind[] = {
225213
};
226214

227215
// must have the same order as enum in lexer.h
216+
// must be sorted according to strcmp
228217
STATIC const char *const tok_kw[] = {
229218
"False",
230219
"None",
231220
"True",
221+
"__debug__",
232222
"and",
233223
"as",
234224
"assert",
@@ -263,7 +253,6 @@ STATIC const char *const tok_kw[] = {
263253
"while",
264254
"with",
265255
"yield",
266-
"__debug__",
267256
};
268257

269258
// This is called with CUR_CHAR() before first hex digit, and should return with
@@ -531,16 +520,18 @@ void mp_lexer_to_next(mp_lexer_t *lex) {
531520
// We also check for __debug__ here and convert it to its value. This is
532521
// so the parser gives a syntax error on, eg, x.__debug__. Otherwise, we
533522
// need to check for this special token in many places in the compiler.
534-
// TODO improve speed of these string comparisons
523+
const char *s = vstr_null_terminated_str(&lex->vstr);
535524
for (size_t i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) {
536-
if (str_strn_equal(tok_kw[i], lex->vstr.buf, lex->vstr.len)) {
537-
if (i == MP_ARRAY_SIZE(tok_kw) - 1) {
538-
// tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__"
525+
int cmp = strcmp(s, tok_kw[i]);
526+
if (cmp == 0) {
527+
lex->tok_kind = MP_TOKEN_KW_FALSE + i;
528+
if (lex->tok_kind == MP_TOKEN_KW___DEBUG__) {
539529
lex->tok_kind = (MP_STATE_VM(mp_optimise_value) == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE);
540-
} else {
541-
lex->tok_kind = MP_TOKEN_KW_FALSE + i;
542530
}
543531
break;
532+
} else if (cmp < 0) {
533+
// Table is sorted and comparison was less-than, so stop searching
534+
break;
544535
}
545536
}
546537

‎py/lexer.h‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ typedef enum _mp_token_kind_t {
6161
MP_TOKEN_KW_FALSE, // 14
6262
MP_TOKEN_KW_NONE,
6363
MP_TOKEN_KW_TRUE,
64+
MP_TOKEN_KW___DEBUG__,
6465
MP_TOKEN_KW_AND,
6566
MP_TOKEN_KW_AS,
6667
MP_TOKEN_KW_ASSERT,
@@ -71,7 +72,7 @@ typedef enum _mp_token_kind_t {
7172
MP_TOKEN_KW_BREAK,
7273
MP_TOKEN_KW_CLASS,
7374
MP_TOKEN_KW_CONTINUE,
74-
MP_TOKEN_KW_DEF, // 23
75+
MP_TOKEN_KW_DEF,
7576
MP_TOKEN_KW_DEL,
7677
MP_TOKEN_KW_ELIF,
7778
MP_TOKEN_KW_ELSE,
@@ -81,7 +82,7 @@ typedef enum _mp_token_kind_t {
8182
MP_TOKEN_KW_FROM,
8283
MP_TOKEN_KW_GLOBAL,
8384
MP_TOKEN_KW_IF,
84-
MP_TOKEN_KW_IMPORT, // 33
85+
MP_TOKEN_KW_IMPORT,
8586
MP_TOKEN_KW_IN,
8687
MP_TOKEN_KW_IS,
8788
MP_TOKEN_KW_LAMBDA,
@@ -91,12 +92,12 @@ typedef enum _mp_token_kind_t {
9192
MP_TOKEN_KW_PASS,
9293
MP_TOKEN_KW_RAISE,
9394
MP_TOKEN_KW_RETURN,
94-
MP_TOKEN_KW_TRY, // 43
95+
MP_TOKEN_KW_TRY,
9596
MP_TOKEN_KW_WHILE,
9697
MP_TOKEN_KW_WITH,
9798
MP_TOKEN_KW_YIELD,
9899

99-
MP_TOKEN_OP_PLUS, // 47
100+
MP_TOKEN_OP_PLUS,
100101
MP_TOKEN_OP_MINUS,
101102
MP_TOKEN_OP_STAR,
102103
MP_TOKEN_OP_DBL_STAR,
@@ -106,7 +107,7 @@ typedef enum _mp_token_kind_t {
106107
MP_TOKEN_OP_LESS,
107108
MP_TOKEN_OP_DBL_LESS,
108109
MP_TOKEN_OP_MORE,
109-
MP_TOKEN_OP_DBL_MORE, // 57
110+
MP_TOKEN_OP_DBL_MORE,
110111
MP_TOKEN_OP_AMPERSAND,
111112
MP_TOKEN_OP_PIPE,
112113
MP_TOKEN_OP_CARET,
@@ -116,7 +117,7 @@ typedef enum _mp_token_kind_t {
116117
MP_TOKEN_OP_DBL_EQUAL,
117118
MP_TOKEN_OP_NOT_EQUAL,
118119

119-
MP_TOKEN_DEL_PAREN_OPEN, // 66
120+
MP_TOKEN_DEL_PAREN_OPEN,
120121
MP_TOKEN_DEL_PAREN_CLOSE,
121122
MP_TOKEN_DEL_BRACKET_OPEN,
122123
MP_TOKEN_DEL_BRACKET_CLOSE,
@@ -126,7 +127,7 @@ typedef enum _mp_token_kind_t {
126127
MP_TOKEN_DEL_COLON,
127128
MP_TOKEN_DEL_PERIOD,
128129
MP_TOKEN_DEL_SEMICOLON,
129-
MP_TOKEN_DEL_AT, // 76
130+
MP_TOKEN_DEL_AT,
130131
MP_TOKEN_DEL_EQUAL,
131132
MP_TOKEN_DEL_PLUS_EQUAL,
132133
MP_TOKEN_DEL_MINUS_EQUAL,
@@ -136,7 +137,7 @@ typedef enum _mp_token_kind_t {
136137
MP_TOKEN_DEL_PERCENT_EQUAL,
137138
MP_TOKEN_DEL_AMPERSAND_EQUAL,
138139
MP_TOKEN_DEL_PIPE_EQUAL,
139-
MP_TOKEN_DEL_CARET_EQUAL, // 86
140+
MP_TOKEN_DEL_CARET_EQUAL,
140141
MP_TOKEN_DEL_DBL_MORE_EQUAL,
141142
MP_TOKEN_DEL_DBL_LESS_EQUAL,
142143
MP_TOKEN_DEL_DBL_STAR_EQUAL,

0 commit comments

Comments
 (0)