Skip to content

Commit c23a2ea

Browse files
dpgeorgedpgeorge
committed
update parser and compiler to align with latest master
1 parent 6cdf816 commit c23a2ea

File tree

4 files changed

+171
-116
lines changed

4 files changed

+171
-116
lines changed

‎py/compile2.c‎

Lines changed: 91 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
11
/*
2-
* This file is part of the Micro Python project, http://micropython.org/
2+
* This file is part of the MicroPython project, http://micropython.org/
33
*
44
* The MIT License (MIT)
55
*
6-
* Copyright (c) 2013-2015 Damien P. George
6+
* Copyright (c) 2013-2016 Damien P. George
77
*
88
* Permission is hereby granted, free of charge, to any person obtaining a copy
99
* of this software and associated documentation files (the "Software"), to deal
@@ -33,9 +33,9 @@
3333
#include "py/scope.h"
3434
#include "py/emit.h"
3535
#include "py/compile.h"
36-
#include "py/smallint.h"
3736
#include "py/runtime.h"
38-
#include "py/builtin.h"
37+
38+
#if MICROPY_ENABLE_COMPILER
3939

4040
// TODO need to mangle __attr names
4141

@@ -80,7 +80,7 @@ typedef struct _compiler_t {
8080

8181
// try to keep compiler clean from nlr
8282
mp_obj_t compile_error; // set to an exception object if there's an error
83-
mp_uint_t compile_error_line; // set to best guess of line of error
83+
size_t compile_error_line; // set to best guess of line of error
8484

8585
uint next_label;
8686

@@ -94,7 +94,7 @@ typedef struct _compiler_t {
9494

9595
mp_uint_t *co_data;
9696

97-
mp_uint_t num_scopes;
97+
size_t num_scopes;
9898
scope_t **scopes;
9999
scope_t *scope_cur;
100100

@@ -112,7 +112,7 @@ typedef struct _compiler_t {
112112
STATIC void compile_error_set_line(compiler_t *comp, const byte *p) {
113113
// if the line of the error is unknown then try to update it from the parse data
114114
if (comp->compile_error_line == 0 && p != NULL && pt_is_any_rule(p)) {
115-
mp_uint_t rule_id, src_line;
115+
size_t rule_id, src_line;
116116
const byte *ptop;
117117
pt_rule_extract(p, &rule_id, &src_line, &ptop);
118118
comp->compile_error_line = src_line;
@@ -147,7 +147,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) {
147147
comp->cur_except_level -= 1;
148148
}
149149

150-
STATIC void scope_new_and_link(compiler_t *comp, mp_uint_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) {
150+
STATIC void scope_new_and_link(compiler_t *comp, size_t scope_idx, scope_kind_t kind, const byte *p, uint emit_options) {
151151
scope_t *scope = scope_new(kind, p, comp->source_file, emit_options);
152152
scope->parent = comp->scope_cur;
153153
comp->scopes[scope_idx] = scope;
@@ -284,14 +284,12 @@ STATIC const byte *c_if_cond(compiler_t *comp, const byte *p, bool jump_if, int
284284
if (jump_if == false) {
285285
EMIT_ARG(jump, label);
286286
}
287-
} else if (pt_is_rule(pt_rule_first(p), PN_testlist_comp)) {
287+
} else {
288+
assert(pt_is_rule(pt_rule_first(p), PN_testlist_comp));
288289
// non-empty tuple, acts as true for the condition
289290
if (jump_if == true) {
290291
EMIT_ARG(jump, label);
291292
}
292-
} else {
293-
// parenthesis around 1 item, is just that item
294-
c_if_cond(comp, pt_rule_first(p), jump_if, label);
295293
}
296294
return pt_next(p);
297295
}
@@ -416,7 +414,6 @@ STATIC void c_assign_tuple(compiler_t *comp, const byte *p_head, const byte *p_t
416414

417415
// assigns top of stack to pn
418416
STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind) {
419-
tail_recursion:
420417
assert(!pt_is_null(p));
421418
if (pt_is_any_id(p)) {
422419
qstr arg;
@@ -459,16 +456,13 @@ STATIC void c_assign(compiler_t *comp, const byte *p, assign_kind_t assign_kind)
459456
if (pt_is_null_with_top(p0, ptop)) {
460457
// empty tuple
461458
goto cannot_assign;
462-
} else if (pt_is_rule(p0, PN_testlist_comp)) {
459+
} else {
460+
assert(pt_is_rule(p0, PN_testlist_comp));
463461
if (assign_kind != ASSIGN_STORE) {
464462
goto bad_aug;
465463
}
466464
p = p0;
467465
goto testlist_comp;
468-
} else {
469-
// parenthesis around 1 item, is just that item
470-
p = p0;
471-
goto tail_recursion;
472466
}
473467
break;
474468
}
@@ -670,6 +664,13 @@ STATIC void compile_funcdef_lambdef_param(compiler_t *comp, const byte *p) {
670664
}
671665

672666
STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte *p, pn_kind_t pn_list_kind) {
667+
// When we call compile_funcdef_lambdef_param below it can compile an arbitrary
668+
// expression for default arguments, which may contain a lambda. The lambda will
669+
// call here in a nested way, so we must save and restore the relevant state.
670+
bool orig_have_star = comp->have_star;
671+
uint16_t orig_num_dict_params = comp->num_dict_params;
672+
uint16_t orig_num_default_params = comp->num_default_params;
673+
673674
// compile default parameters
674675
comp->have_star = false;
675676
comp->num_dict_params = 0;
@@ -689,6 +690,11 @@ STATIC void compile_funcdef_lambdef(compiler_t *comp, scope_t *scope, const byte
689690

690691
// make the function
691692
close_over_variables_etc(comp, scope, comp->num_default_params, comp->num_dict_params);
693+
694+
// restore state
695+
comp->have_star = orig_have_star;
696+
comp->num_dict_params = orig_num_dict_params;
697+
comp->num_default_params = orig_num_default_params;
692698
}
693699

694700
// leaves function object on stack
@@ -897,8 +903,11 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) {
897903
goto cannot_delete;
898904
}
899905
} else if (pt_is_rule(p, PN_atom_paren)) {
900-
p = pt_rule_first(p);
901-
if (pt_is_rule(p, PN_testlist_comp)) {
906+
if (pt_is_rule_empty(p)) {
907+
goto cannot_delete;
908+
} else {
909+
p = pt_rule_first(p);
910+
assert(pt_is_rule(p, PN_testlist_comp));
902911
// TODO perhaps factorise testlist_comp code with other uses of PN_testlist_comp
903912
// or, simplify the logic here my making the parser simplify everything to a list
904913
const byte *p0 = pt_rule_first(p);
@@ -923,12 +932,9 @@ STATIC void c_del_stmt(compiler_t *comp, const byte *p) {
923932
// sequence with 2 items
924933
c_del_stmt(comp, p1);
925934
}
926-
} else {
927-
// tuple with 1 element
928-
c_del_stmt(comp, p);
929935
}
930936
} else {
931-
// TODO is there anything else to implement?
937+
// some arbitrary statment that we can't delete (eg del 1)
932938
goto cannot_delete;
933939
}
934940

@@ -1062,7 +1068,7 @@ STATIC void do_import_name(compiler_t *comp, const byte *p, qstr *q_base) {
10621068
}
10631069
qstr qst;
10641070
p2 = pt_extract_id(p2, &qst);
1065-
mp_uint_t str_src_len;
1071+
size_t str_src_len;
10661072
const byte *str_src = qstr_data(qst, &str_src_len);
10671073
memcpy(str_dest, str_src, str_src_len);
10681074
str_dest += str_src_len;
@@ -1495,6 +1501,19 @@ STATIC void compile_for_stmt(compiler_t *comp, const byte *p, const byte *ptop)
14951501
}
14961502
}
14971503
}
1504+
// arguments must be able to be compiled as standard expressions
1505+
if (pt_is_any_rule(p_start)) {
1506+
int k = pt_rule_extract_rule_id(p_start);
1507+
if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) {
1508+
goto optimise_fail;
1509+
}
1510+
}
1511+
if (pt_is_any_rule(p_end)) {
1512+
int k = pt_rule_extract_rule_id(p_end);
1513+
if (k == PN_arglist_star || k == PN_arglist_dbl_star || k == PN_argument) {
1514+
goto optimise_fail;
1515+
}
1516+
}
14981517
// can optimise
14991518
const byte *p_body = p_it_top;
15001519
const byte *p_else = pt_next(p_body);
@@ -2221,7 +2240,8 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop
22212240
if (pt_is_null_with_top(p, ptop)) {
22222241
// an empty tuple
22232242
c_tuple(comp, NULL, NULL, NULL);
2224-
} else if (pt_is_rule(p, PN_testlist_comp)) {
2243+
} else {
2244+
assert(pt_is_rule(p, PN_testlist_comp));
22252245
p = pt_rule_first(p);
22262246
const byte *p1 = pt_next(p);
22272247
if (pt_is_rule(p1, PN_testlist_comp_3b) || pt_is_rule(p1, PN_testlist_comp_3c)) {
@@ -2234,9 +2254,6 @@ STATIC void compile_atom_paren(compiler_t *comp, const byte *p, const byte *ptop
22342254
// tuple with 2 items
22352255
c_tuple(comp, NULL, p, ptop);
22362256
}
2237-
} else {
2238-
// parenthesis around a single item, is just that item
2239-
compile_node(comp, p);
22402257
}
22412258
}
22422259

@@ -2499,7 +2516,7 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) {
24992516
EMIT_ARG(load_const_obj, mp_const_none);
25002517
} else {
25012518
qstr qst = p[1] | (p[2] << 8);
2502-
mp_uint_t len;
2519+
size_t len;
25032520
const byte *data = qstr_data(qst, &len);
25042521
EMIT_ARG(load_const_obj, mp_obj_new_bytes(data, len));
25052522
}
@@ -2510,13 +2527,13 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) {
25102527
compile_load_id(comp, qst);
25112528
return p;
25122529
} else if (*p == MP_PT_CONST_OBJECT) {
2513-
mp_uint_t idx;
2530+
size_t idx;
25142531
p = pt_extract_const_obj(p, &idx);
25152532
EMIT_ARG(load_const_obj, (mp_obj_t)comp->co_data[idx]);
25162533
return p;
25172534
} else {
25182535
assert(*p >= MP_PT_RULE_BASE);
2519-
mp_uint_t rule_id, src_line;
2536+
size_t rule_id, src_line;
25202537
const byte *ptop;
25212538
p = pt_rule_extract(p, &rule_id, &src_line, &ptop);
25222539
EMIT_ARG(set_source_line, src_line);
@@ -2542,7 +2559,12 @@ STATIC const byte *compile_node(compiler_t *comp, const byte *p) {
25422559

25432560
STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_kind_t pn_name, pn_kind_t pn_star, pn_kind_t pn_dbl_star) {
25442561
(void)pn_dbl_star;
2545-
// TODO verify that *k and **k are last etc
2562+
// check that **kw is last
2563+
if ((comp->scope_cur->scope_flags & MP_SCOPE_FLAG_VARKEYWORDS) != 0) {
2564+
compile_syntax_error(comp, p, "invalid syntax");
2565+
return;
2566+
}
2567+
25462568
qstr param_name = MP_QSTR_NULL;
25472569
uint param_flag = ID_FLAG_IS_PARAM;
25482570
if (pt_is_any_id(p)) {
@@ -2565,6 +2587,11 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, const byte *p, pn_
25652587
comp->scope_cur->num_pos_args += 1;
25662588
}
25672589
} else if (pt_is_rule(p, pn_star)) {
2590+
if (comp->have_star) {
2591+
// more than one star
2592+
compile_syntax_error(comp, p, "invalid syntax");
2593+
return;
2594+
}
25682595
comp->have_star = true;
25692596
param_flag = ID_FLAG_IS_PARAM | ID_FLAG_IS_STAR_PARAM;
25702597
if (pt_is_rule_empty(p)) {
@@ -2655,9 +2682,16 @@ STATIC void compile_scope_func_annotations(compiler_t *comp, const byte *p) {
26552682
}
26562683
#endif // MICROPY_EMIT_NATIVE
26572684

2658-
STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const byte *p_inner_expr, int l_top, int for_depth) {
2685+
STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_comp_for, const byte *p_comp_for_top, const byte *p_inner_expr, int for_depth) {
2686+
uint l_top = comp_next_label(comp);
2687+
uint l_end = comp_next_label(comp);
2688+
EMIT_ARG(label_assign, l_top);
2689+
EMIT_ARG(for_iter, l_end);
2690+
c_assign(comp, p_comp_for, ASSIGN_STORE);
2691+
const byte *p_iter = pt_next(pt_next(p_comp_for));
2692+
26592693
tail_recursion:
2660-
if (p_iter == NULL) {
2694+
if (p_iter == p_comp_for_top) {
26612695
// no more nested if/for; compile inner expression
26622696
compile_node(comp, p_inner_expr);
26632697
if (comp->scope_cur->kind == SCOPE_LIST_COMP) {
@@ -2674,31 +2708,23 @@ STATIC void compile_scope_comp_iter(compiler_t *comp, const byte *p_iter, const
26742708
}
26752709
} else if (pt_is_rule(p_iter, PN_comp_if)) {
26762710
// if condition
2677-
const byte *ptop;
2678-
const byte *p0 = pt_rule_extract_top(p_iter, &ptop);
2711+
const byte *p0 = pt_rule_extract_top(p_iter, &p_comp_for_top);
26792712
p_iter = c_if_cond(comp, p0, false, l_top);
2680-
if (p_iter == ptop) {
2681-
p_iter = NULL;
2682-
}
26832713
goto tail_recursion;
26842714
} else {
26852715
assert(pt_is_rule(p_iter, PN_comp_for)); // should be
26862716
// for loop
26872717
const byte *ptop;
26882718
const byte *p0 = pt_rule_extract_top(p_iter, &ptop);
26892719
p0 = pt_next(p0); // skip scope index
2690-
const byte *p2 = compile_node(comp, pt_next(p0));
2691-
uint l_end2 = comp_next_label(comp);
2692-
uint l_top2 = comp_next_label(comp);
2720+
compile_node(comp, pt_next(p0));
26932721
EMIT(get_iter);
2694-
EMIT_ARG(label_assign, l_top2);
2695-
EMIT_ARG(for_iter, l_end2);
2696-
c_assign(comp, p0, ASSIGN_STORE);
2697-
compile_scope_comp_iter(comp, p2 == ptop ? NULL : p2, p_inner_expr, l_top2, for_depth + 1);
2698-
EMIT_ARG(jump, l_top2);
2699-
EMIT_ARG(label_assign, l_end2);
2700-
EMIT(for_iter_end);
2722+
compile_scope_comp_iter(comp, p0, ptop, p_inner_expr, for_depth + 1);
27012723
}
2724+
2725+
EMIT_ARG(jump, l_top);
2726+
EMIT_ARG(label_assign, l_end);
2727+
EMIT(for_iter_end);
27022728
}
27032729

27042730
#if 0
@@ -2865,20 +2891,8 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) {
28652891
#endif
28662892
}
28672893

2868-
uint l_end = comp_next_label(comp);
2869-
uint l_top = comp_next_label(comp);
28702894
compile_load_id(comp, qstr_arg);
2871-
EMIT_ARG(label_assign, l_top);
2872-
EMIT_ARG(for_iter, l_end);
2873-
c_assign(comp, p_comp_for, ASSIGN_STORE);
2874-
const byte *p_comp_for_p2 = pt_next(pt_next(p_comp_for));
2875-
if (p_comp_for_p2 == p_comp_for_top) {
2876-
p_comp_for_p2 = NULL;
2877-
}
2878-
compile_scope_comp_iter(comp, p_comp_for_p2, p, l_top, 0);
2879-
EMIT_ARG(jump, l_top);
2880-
EMIT_ARG(label_assign, l_end);
2881-
EMIT(for_iter_end);
2895+
compile_scope_comp_iter(comp, p_comp_for, p_comp_for_top, p, 0);
28822896

28832897
if (scope->kind == SCOPE_GEN_EXPR) {
28842898
EMIT_ARG(load_const_tok, MP_TOKEN_KW_NONE);
@@ -3063,7 +3077,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind
30633077
}
30643078

30653079
if (comp->pass > MP_PASS_SCOPE) {
3066-
EMIT_INLINE_ASM(end_pass);
3080+
EMIT_INLINE_ASM_ARG(end_pass, 0);
30673081
}
30683082

30693083
if (comp->compile_error != MP_OBJ_NULL) {
@@ -3154,7 +3168,10 @@ STATIC void scope_compute_things(scope_t *scope) {
31543168
}
31553169
}
31563170

3157-
mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
3171+
#if !MICROPY_PERSISTENT_CODE_SAVE
3172+
STATIC
3173+
#endif
3174+
mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
31583175
// put compiler state on the stack, it's relatively small
31593176
compiler_t comp_state = {0};
31603177
compiler_t *comp = &comp_state;
@@ -3351,7 +3368,14 @@ mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt
33513368
if (comp->compile_error != MP_OBJ_NULL) {
33523369
nlr_raise(comp->compile_error);
33533370
} else {
3354-
// return function that executes the outer module
3355-
return mp_make_function_from_raw_code(outer_raw_code, MP_OBJ_NULL, MP_OBJ_NULL);
3371+
return outer_raw_code;
33563372
}
33573373
}
3374+
3375+
mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, uint emit_opt, bool is_repl) {
3376+
mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, emit_opt, is_repl);
3377+
// return function that executes the outer module
3378+
return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL);
3379+
}
3380+
3381+
#endif // MICROPY_ENABLE_COMPILER

‎py/emit.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -269,8 +269,6 @@ typedef struct _emit_inline_asm_t emit_inline_asm_t;
269269
typedef struct _emit_inline_asm_method_table_t {
270270
void (*start_pass)(emit_inline_asm_t *emit, pass_kind_t pass, scope_t *scope, mp_obj_t *error_slot);
271271
void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig);
272-
mp_uint_t (*count_params)(emit_inline_asm_t *emit, mp_uint_t n_params, mp_parse_node_t *pn_params);
273-
void (*end_pass)(emit_inline_asm_t *emit, mp_uint_t type_sig);
274272
mp_uint_t (*count_params)(emit_inline_asm_t *emit, const byte *p, const byte *ptop);
275273
bool (*label)(emit_inline_asm_t *emit, mp_uint_t label_num, qstr label_id);
276274
void (*align)(emit_inline_asm_t *emit, mp_uint_t align);

0 commit comments

Comments
 (0)