Skip to content

Commit b25ef4d

Browse files
dpgeorgedpgeorge
committed
Merge pull request adafruit#314 from pfalcon/parse-node-refactor
parse: Refactor parse node encoding to support full range of small ints.
2 parents bbf0e2f + 56e5ef2 commit b25ef4d

5 files changed

Lines changed: 84 additions & 46 deletions

File tree

‎py/compile.c‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ mp_parse_node_t fold_constants(mp_parse_node_t pn) {
8686
switch (MP_PARSE_NODE_STRUCT_KIND(pns)) {
8787
case PN_shift_expr:
8888
if (n == 3 && MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[0]) && MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[2])) {
89-
int arg0 = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]);
90-
int arg1 = MP_PARSE_NODE_LEAF_ARG(pns->nodes[2]);
89+
int arg0 = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[0]);
90+
int arg1 = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[2]);
9191
if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[1], MP_TOKEN_OP_DBL_LESS)) {
9292
#if MICROPY_EMIT_CPYTHON
9393
// can overflow; enabled only to compare with CPython
@@ -105,8 +105,8 @@ mp_parse_node_t fold_constants(mp_parse_node_t pn) {
105105
case PN_arith_expr:
106106
// overflow checking here relies on SMALL_INT being strictly smaller than machine_int_t
107107
if (n == 3 && MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[0]) && MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[2])) {
108-
machine_int_t arg0 = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]);
109-
machine_int_t arg1 = MP_PARSE_NODE_LEAF_ARG(pns->nodes[2]);
108+
machine_int_t arg0 = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[0]);
109+
machine_int_t arg1 = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[2]);
110110
machine_int_t res;
111111
if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[1], MP_TOKEN_OP_PLUS)) {
112112
res = arg0 + arg1;
@@ -125,8 +125,8 @@ mp_parse_node_t fold_constants(mp_parse_node_t pn) {
125125

126126
case PN_term:
127127
if (n == 3 && MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[0]) && MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[2])) {
128-
int arg0 = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]);
129-
int arg1 = MP_PARSE_NODE_LEAF_ARG(pns->nodes[2]);
128+
int arg0 = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[0]);
129+
int arg1 = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[2]);
130130
if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[1], MP_TOKEN_OP_STAR)) {
131131
#if MICROPY_EMIT_CPYTHON
132132
// can overflow; enabled only to compare with CPython
@@ -149,7 +149,7 @@ mp_parse_node_t fold_constants(mp_parse_node_t pn) {
149149

150150
case PN_factor_2:
151151
if (MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[1])) {
152-
machine_int_t arg = MP_PARSE_NODE_LEAF_ARG(pns->nodes[1]);
152+
machine_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[1]);
153153
if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[0], MP_TOKEN_OP_PLUS)) {
154154
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_SMALL_INT, arg);
155155
} else if (MP_PARSE_NODE_IS_TOKEN_KIND(pns->nodes[0], MP_TOKEN_OP_MINUS)) {
@@ -169,10 +169,10 @@ mp_parse_node_t fold_constants(mp_parse_node_t pn) {
169169
if (MP_PARSE_NODE_IS_SMALL_INT(pns->nodes[0]) && MP_PARSE_NODE_IS_NULL(pns->nodes[1]) && !MP_PARSE_NODE_IS_NULL(pns->nodes[2])) {
170170
mp_parse_node_struct_t* pns2 = (mp_parse_node_struct_t*)pns->nodes[2];
171171
if (MP_PARSE_NODE_IS_SMALL_INT(pns2->nodes[0])) {
172-
int power = MP_PARSE_NODE_LEAF_ARG(pns2->nodes[0]);
172+
int power = MP_PARSE_NODE_LEAF_SMALL_INT(pns2->nodes[0]);
173173
if (power >= 0) {
174174
int ans = 1;
175-
int base = MP_PARSE_NODE_LEAF_ARG(pns->nodes[0]);
175+
int base = MP_PARSE_NODE_LEAF_SMALL_INT(pns->nodes[0]);
176176
for (; power > 0; power--) {
177177
ans *= base;
178178
}
@@ -320,10 +320,14 @@ STATIC void cpython_c_print_quoted_str(vstr_t *vstr, qstr qstr, bool bytes) {
320320

321321
STATIC void cpython_c_tuple_emit_const(compiler_t *comp, mp_parse_node_t pn, vstr_t *vstr) {
322322
assert(MP_PARSE_NODE_IS_LEAF(pn));
323+
if (MP_PARSE_NODE_IS_SMALL_INT(pn)) {
324+
vstr_printf(vstr, INT_FMT, MP_PARSE_NODE_LEAF_SMALL_INT(pn));
325+
return;
326+
}
327+
323328
int arg = MP_PARSE_NODE_LEAF_ARG(pn);
324329
switch (MP_PARSE_NODE_LEAF_KIND(pn)) {
325330
case MP_PARSE_NODE_ID: assert(0);
326-
case MP_PARSE_NODE_SMALL_INT: vstr_printf(vstr, "%d", arg); break;
327331
case MP_PARSE_NODE_INTEGER: vstr_printf(vstr, "%s", qstr_str(arg)); break;
328332
case MP_PARSE_NODE_DECIMAL: vstr_printf(vstr, "%s", qstr_str(arg)); break;
329333
case MP_PARSE_NODE_STRING: cpython_c_print_quoted_str(vstr, arg, false); break;
@@ -421,11 +425,11 @@ void compile_generic_tuple(compiler_t *comp, mp_parse_node_struct_t *pns) {
421425

422426
STATIC bool node_is_const_false(mp_parse_node_t pn) {
423427
return MP_PARSE_NODE_IS_TOKEN_KIND(pn, MP_TOKEN_KW_FALSE);
424-
// untested: || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_ARG(pn) == 1);
428+
// untested: || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_SMALL_INT(pn) == 0);
425429
}
426430

427431
STATIC bool node_is_const_true(mp_parse_node_t pn) {
428-
return MP_PARSE_NODE_IS_TOKEN_KIND(pn, MP_TOKEN_KW_TRUE) || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_ARG(pn) == 1);
432+
return MP_PARSE_NODE_IS_TOKEN_KIND(pn, MP_TOKEN_KW_TRUE) || (MP_PARSE_NODE_IS_SMALL_INT(pn) && MP_PARSE_NODE_LEAF_SMALL_INT(pn) == 1);
429433
}
430434

431435
#if MICROPY_EMIT_CPYTHON
@@ -1464,7 +1468,8 @@ void compile_for_stmt_optimised_range(compiler_t *comp, mp_parse_node_t pn_var,
14641468
// compile: if var <cond> end: goto top
14651469
compile_node(comp, pn_var);
14661470
compile_node(comp, pn_end);
1467-
if (MP_PARSE_NODE_LEAF_ARG(pn_step) >= 0) {
1471+
assert(MP_PARSE_NODE_IS_SMALL_INT(pn_step));
1472+
if (MP_PARSE_NODE_LEAF_SMALL_INT(pn_step) >= 0) {
14681473
EMIT_ARG(binary_op, RT_BINARY_OP_LESS);
14691474
} else {
14701475
EMIT_ARG(binary_op, RT_BINARY_OP_MORE);
@@ -2514,11 +2519,13 @@ STATIC compile_function_t compile_function[] = {
25142519
void compile_node(compiler_t *comp, mp_parse_node_t pn) {
25152520
if (MP_PARSE_NODE_IS_NULL(pn)) {
25162521
// pass
2522+
} else if (MP_PARSE_NODE_IS_SMALL_INT(pn)) {
2523+
machine_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn);
2524+
EMIT_ARG(load_const_small_int, arg);
25172525
} else if (MP_PARSE_NODE_IS_LEAF(pn)) {
2518-
machine_int_t arg = MP_PARSE_NODE_LEAF_ARG(pn);
2526+
machine_uint_t arg = MP_PARSE_NODE_LEAF_ARG(pn);
25192527
switch (MP_PARSE_NODE_LEAF_KIND(pn)) {
25202528
case MP_PARSE_NODE_ID: EMIT_ARG(load_id, arg); break;
2521-
case MP_PARSE_NODE_SMALL_INT: EMIT_ARG(load_const_small_int, arg); break;
25222529
case MP_PARSE_NODE_INTEGER: EMIT_ARG(load_const_int, arg); break;
25232530
case MP_PARSE_NODE_DECIMAL: EMIT_ARG(load_const_dec, arg); break;
25242531
case MP_PARSE_NODE_STRING: EMIT_ARG(load_const_str, arg, false); break;

‎py/emitinlinethumb.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ STATIC int get_arg_i(qstr op, mp_parse_node_t *pn_args, int wanted_arg_num, int
110110
printf("SyntaxError: '%s' expects an integer in position %d\n", qstr_str(op), wanted_arg_num);
111111
return 0;
112112
}
113-
int i = MP_PARSE_NODE_LEAF_ARG(pn_args[wanted_arg_num]);
113+
int i = MP_PARSE_NODE_LEAF_SMALL_INT(pn_args[wanted_arg_num]);
114114
if ((i & (~fit_mask)) != 0) {
115115
printf("SyntaxError: '%s' integer 0x%x does not fit in mask 0x%x\n", qstr_str(op), i, fit_mask);
116116
return 0;

‎py/parse.c‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,10 @@ STATIC void pop_rule(parser_t *parser, const rule_t **rule, uint *arg_i, uint *s
125125
}
126126

127127
mp_parse_node_t mp_parse_node_new_leaf(machine_int_t kind, machine_int_t arg) {
128-
return (mp_parse_node_t)(kind | (arg << 4));
128+
if (kind == MP_PARSE_NODE_SMALL_INT) {
129+
return (mp_parse_node_t)(kind | (arg << 1));
130+
}
131+
return (mp_parse_node_t)(kind | (arg << 5));
129132
}
130133

131134
//int num_parse_nodes_allocated = 0;
@@ -171,11 +174,13 @@ void mp_parse_node_print(mp_parse_node_t pn, int indent) {
171174
}
172175
if (MP_PARSE_NODE_IS_NULL(pn)) {
173176
printf("NULL\n");
177+
} else if (MP_PARSE_NODE_IS_SMALL_INT(pn)) {
178+
machine_int_t arg = MP_PARSE_NODE_LEAF_SMALL_INT(pn);
179+
printf("int(" INT_FMT ")\n", arg);
174180
} else if (MP_PARSE_NODE_IS_LEAF(pn)) {
175-
machine_int_t arg = MP_PARSE_NODE_LEAF_ARG(pn);
181+
machine_uint_t arg = MP_PARSE_NODE_LEAF_ARG(pn);
176182
switch (MP_PARSE_NODE_LEAF_KIND(pn)) {
177183
case MP_PARSE_NODE_ID: printf("id(%s)\n", qstr_str(arg)); break;
178-
case MP_PARSE_NODE_SMALL_INT: printf("int(" INT_FMT ")\n", arg); break;
179184
case MP_PARSE_NODE_INTEGER: printf("int(%s)\n", qstr_str(arg)); break;
180185
case MP_PARSE_NODE_DECIMAL: printf("dec(%s)\n", qstr_str(arg)); break;
181186
case MP_PARSE_NODE_STRING: printf("str(%s)\n", qstr_str(arg)); break;

‎py/parse.h‎

Lines changed: 29 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,30 @@ struct _mp_lexer_t;
22

33
// a mp_parse_node_t is:
44
// - 0000...0000: no node
5-
// - xxxx...0001: an identifier; bits 4 and above are the qstr
6-
// - xxxx...0011: a small integer; bits 4 and above are the signed value, 2's complement
7-
// - xxxx...0101: an integer; bits 4 and above are the qstr holding the value
8-
// - xxxx...0111: a decimal; bits 4 and above are the qstr holding the value
9-
// - xxxx...1001: a string; bits 4 and above are the qstr holding the value
10-
// - xxxx...1011: a string with triple quotes; bits 4 and above are the qstr holding the value
11-
// - xxxx...1101: a token; bits 4 and above are mp_token_kind_t
12-
// - xxxx...xxx0: pointer to mp_parse_node_struct_t
5+
// - xxxx...xxx1: a small integer; bits 1 and above are the signed value, 2's complement
6+
// - xxxx...xx00: pointer to mp_parse_node_struct_t
7+
// - xx...x00010: an identifier; bits 5 and above are the qstr
8+
// - xx...x00110: an integer; bits 5 and above are the qstr holding the value
9+
// - xx...x01010: a decimal; bits 5 and above are the qstr holding the value
10+
// - xx...x01110: a string; bits 5 and above are the qstr holding the value
11+
// - xx...x10010: a string with triple quotes; bits 5 and above are the qstr holding the value
12+
// - xx...x10110: a token; bits 5 and above are mp_token_kind_t
1313

14-
// makes sure the top 5 bits of x are all cleared (positive number) or all set (negavite number)
14+
// TODO: these can now be unified with MP_OBJ_FITS_SMALL_INT(x)
15+
// makes sure the top 2 bits of x are all cleared (positive number) or all set (negavite number)
1516
// these macros can probably go somewhere else because they are used more than just in the parser
16-
#define MP_UINT_HIGH_5_BITS (~((~((machine_uint_t)0)) >> 5))
17+
#define MP_UINT_HIGH_2_BITS (~((~((machine_uint_t)0)) >> 2))
1718
// parser's small ints are different from VM small int
18-
#define MP_PARSE_FITS_SMALL_INT(x) (((((machine_uint_t)(x)) & MP_UINT_HIGH_5_BITS) == 0) || ((((machine_uint_t)(x)) & MP_UINT_HIGH_5_BITS) == MP_UINT_HIGH_5_BITS))
19+
#define MP_PARSE_FITS_SMALL_INT(x) (((((machine_uint_t)(x)) & MP_UINT_HIGH_2_BITS) == 0) || ((((machine_uint_t)(x)) & MP_UINT_HIGH_2_BITS) == MP_UINT_HIGH_2_BITS))
1920

2021
#define MP_PARSE_NODE_NULL (0)
21-
#define MP_PARSE_NODE_ID (0x1)
22-
#define MP_PARSE_NODE_SMALL_INT (0x3)
23-
#define MP_PARSE_NODE_INTEGER (0x5)
24-
#define MP_PARSE_NODE_DECIMAL (0x7)
25-
#define MP_PARSE_NODE_STRING (0x9)
26-
#define MP_PARSE_NODE_BYTES (0xb)
27-
#define MP_PARSE_NODE_TOKEN (0xd)
22+
#define MP_PARSE_NODE_SMALL_INT (0x1)
23+
#define MP_PARSE_NODE_ID (0x02)
24+
#define MP_PARSE_NODE_INTEGER (0x06)
25+
#define MP_PARSE_NODE_DECIMAL (0x0a)
26+
#define MP_PARSE_NODE_STRING (0x0e)
27+
#define MP_PARSE_NODE_BYTES (0x12)
28+
#define MP_PARSE_NODE_TOKEN (0x16)
2829

2930
typedef machine_uint_t mp_parse_node_t; // must be pointer size
3031

@@ -38,18 +39,19 @@ typedef struct _mp_parse_node_struct_t {
3839
// some of these evaluate their argument more than once
3940

4041
#define MP_PARSE_NODE_IS_NULL(pn) ((pn) == MP_PARSE_NODE_NULL)
41-
#define MP_PARSE_NODE_IS_LEAF(pn) ((pn) & 1)
42-
#define MP_PARSE_NODE_IS_STRUCT(pn) ((pn) != MP_PARSE_NODE_NULL && ((pn) & 1) == 0)
43-
#define MP_PARSE_NODE_IS_STRUCT_KIND(pn, k) ((pn) != MP_PARSE_NODE_NULL && ((pn) & 1) == 0 && MP_PARSE_NODE_STRUCT_KIND((mp_parse_node_struct_t*)(pn)) == (k))
42+
#define MP_PARSE_NODE_IS_LEAF(pn) ((pn) & 3)
43+
#define MP_PARSE_NODE_IS_STRUCT(pn) ((pn) != MP_PARSE_NODE_NULL && ((pn) & 3) == 0)
44+
#define MP_PARSE_NODE_IS_STRUCT_KIND(pn, k) ((pn) != MP_PARSE_NODE_NULL && ((pn) & 3) == 0 && MP_PARSE_NODE_STRUCT_KIND((mp_parse_node_struct_t*)(pn)) == (k))
4445

45-
#define MP_PARSE_NODE_IS_ID(pn) (((pn) & 0xf) == MP_PARSE_NODE_ID)
46-
#define MP_PARSE_NODE_IS_SMALL_INT(pn) (((pn) & 0xf) == MP_PARSE_NODE_SMALL_INT)
47-
#define MP_PARSE_NODE_IS_TOKEN(pn) (((pn) & 0xf) == MP_PARSE_NODE_TOKEN)
48-
#define MP_PARSE_NODE_IS_TOKEN_KIND(pn, k) ((pn) == (MP_PARSE_NODE_TOKEN | (k << 4)))
46+
#define MP_PARSE_NODE_IS_SMALL_INT(pn) (((pn) & 0x1) == MP_PARSE_NODE_SMALL_INT)
47+
#define MP_PARSE_NODE_IS_ID(pn) (((pn) & 0x1f) == MP_PARSE_NODE_ID)
48+
#define MP_PARSE_NODE_IS_TOKEN(pn) (((pn) & 0x1f) == MP_PARSE_NODE_TOKEN)
49+
#define MP_PARSE_NODE_IS_TOKEN_KIND(pn, k) ((pn) == (MP_PARSE_NODE_TOKEN | ((k) << 5)))
4950

50-
#define MP_PARSE_NODE_LEAF_KIND(pn) ((pn) & 0xf)
51+
#define MP_PARSE_NODE_LEAF_KIND(pn) ((pn) & 0x1f)
5152
// TODO should probably have int and uint versions of this macro
52-
#define MP_PARSE_NODE_LEAF_ARG(pn) (((machine_int_t)(pn)) >> 4)
53+
#define MP_PARSE_NODE_LEAF_ARG(pn) (((machine_uint_t)(pn)) >> 5)
54+
#define MP_PARSE_NODE_LEAF_SMALL_INT(pn) (((machine_int_t)(pn)) >> 1)
5355
#define MP_PARSE_NODE_STRUCT_KIND(pns) ((pns)->kind_num_nodes & 0xff)
5456
#define MP_PARSE_NODE_STRUCT_NUM_NODES(pns) ((pns)->kind_num_nodes >> 8)
5557

‎tests/basics/int-small.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,29 @@
11
# This tests small int range for 32-bit machine
22

3+
# Small ints are variable-length encoded in MicroPython, so first
4+
# test that encoding works as expected.
5+
6+
print(0)
7+
print(1)
8+
print(-1)
9+
# Value is split in 7-bit "subwords", and taking into account that all
10+
# ints in Python are signed, there're 6 bits of magnitude. So, around 2^6
11+
# there's "turning point"
12+
print(63)
13+
print(64)
14+
print(65)
15+
print(-63)
16+
print(-64)
17+
print(-65)
18+
# Maximum values of small ints on 32-bit platform
19+
print(1073741823)
20+
# Per python semantics, lexical integer is without a sign (i.e. positive)
21+
# and '-' is unary minus operation applied to it. That's why -1073741824
22+
# (min two-complement's negative value) is not allowed.
23+
print(-1073741823)
24+
25+
# Operations tests
26+
327
a = 0x3fffff
428
print(a)
529
a *= 0x10

0 commit comments

Comments
 (0)