Skip to content

Commit 1b82e9a

Browse files
dpgeorgedpgeorge
committed
py: Improve handling of memory error in parser.
Parser shouldn't raise exceptions, so needs to check when memory allocation fails. This patch does that for the initial set up of the parser state. Also, we now put the parser object on the stack. It's small enough to go there instead of on the heap. This partially addresses issue adafruit#558.
1 parent ad6178b commit 1b82e9a

2 files changed

Lines changed: 53 additions & 47 deletions

File tree

‎py/misc.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ typedef unsigned int uint;
5050
// TODO make a lazy m_renew that can increase by a smaller amount than requested (but by at least 1 more element)
5151

5252
#define m_new(type, num) ((type*)(m_malloc(sizeof(type) * (num))))
53+
#define m_new_maybe(type, num) ((type*)(m_malloc_maybe(sizeof(type) * (num))))
5354
#define m_new0(type, num) ((type*)(m_malloc0(sizeof(type) * (num))))
5455
#define m_new_obj(type) (m_new(type, 1))
5556
#define m_new_obj_var(obj_type, var_type, var_num) ((obj_type*)m_malloc(sizeof(obj_type) + sizeof(var_type) * (var_num)))

‎py/parse.c‎

Lines changed: 52 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -344,21 +344,26 @@ STATIC void push_result_rule(parser_t *parser, int src_line, const rule_t *rule,
344344

345345
mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_parse_error_kind_t *parse_error_kind_out) {
346346

347-
// allocate memory for the parser and its stacks
347+
// initialise parser and allocate memory for its stacks
348348

349-
parser_t *parser = m_new_obj(parser_t);
349+
parser_t parser;
350350

351-
parser->had_memory_error = false;
351+
parser.had_memory_error = false;
352352

353-
parser->rule_stack_alloc = MP_ALLOC_PARSE_RULE_INIT;
354-
parser->rule_stack_top = 0;
355-
parser->rule_stack = m_new(rule_stack_t, parser->rule_stack_alloc);
353+
parser.rule_stack_alloc = MP_ALLOC_PARSE_RULE_INIT;
354+
parser.rule_stack_top = 0;
355+
parser.rule_stack = m_new_maybe(rule_stack_t, parser.rule_stack_alloc);
356356

357-
parser->result_stack_alloc = MP_ALLOC_PARSE_RESULT_INIT;
358-
parser->result_stack_top = 0;
359-
parser->result_stack = m_new(mp_parse_node_t, parser->result_stack_alloc);
357+
parser.result_stack_alloc = MP_ALLOC_PARSE_RESULT_INIT;
358+
parser.result_stack_top = 0;
359+
parser.result_stack = m_new_maybe(mp_parse_node_t, parser.result_stack_alloc);
360360

361-
parser->lexer = lex;
361+
parser.lexer = lex;
362+
363+
// check if we could allocate the stacks
364+
if (parser.rule_stack == NULL || parser.result_stack == NULL) {
365+
goto memory_error;
366+
}
362367

363368
// work out the top-level rule to use, and push it on the stack
364369
int top_level_rule;
@@ -367,7 +372,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
367372
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
368373
default: top_level_rule = RULE_file_input;
369374
}
370-
push_rule(parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0);
375+
push_rule(&parser, mp_lexer_cur(lex)->src_line, rules[top_level_rule], 0);
371376

372377
// parse!
373378

@@ -381,17 +386,17 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
381386

382387
for (;;) {
383388
next_rule:
384-
if (parser->rule_stack_top == 0 || parser->had_memory_error) {
389+
if (parser.rule_stack_top == 0 || parser.had_memory_error) {
385390
break;
386391
}
387392

388-
pop_rule(parser, &rule, &i, &rule_src_line);
393+
pop_rule(&parser, &rule, &i, &rule_src_line);
389394
n = rule->act & RULE_ACT_ARG_MASK;
390395

391396
/*
392397
// debugging
393-
printf("depth=%d ", parser->rule_stack_top);
394-
for (int j = 0; j < parser->rule_stack_top; ++j) {
398+
printf("depth=%d ", parser.rule_stack_top);
399+
for (int j = 0; j < parser.rule_stack_top; ++j) {
395400
printf(" ");
396401
}
397402
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
@@ -408,29 +413,29 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
408413
switch (rule->arg[i] & RULE_ARG_KIND_MASK) {
409414
case RULE_ARG_TOK:
410415
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
411-
push_result_token(parser, lex);
416+
push_result_token(&parser, lex);
412417
mp_lexer_to_next(lex);
413418
goto next_rule;
414419
}
415420
break;
416421
case RULE_ARG_RULE:
417-
push_rule(parser, rule_src_line, rule, i + 1); // save this or-rule
418-
push_rule_from_arg(parser, rule->arg[i]); // push child of or-rule
422+
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
423+
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
419424
goto next_rule;
420425
default:
421426
assert(0);
422427
}
423428
}
424429
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
425430
if (mp_lexer_is_kind(lex, rule->arg[i] & RULE_ARG_ARG_MASK)) {
426-
push_result_token(parser, lex);
431+
push_result_token(&parser, lex);
427432
mp_lexer_to_next(lex);
428433
} else {
429434
backtrack = true;
430435
goto next_rule;
431436
}
432437
} else {
433-
push_rule_from_arg(parser, rule->arg[i]);
438+
push_rule_from_arg(&parser, rule->arg[i]);
434439
}
435440
break;
436441

@@ -441,7 +446,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
441446
assert(i > 0);
442447
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
443448
// an optional rule that failed, so continue with next arg
444-
push_result_node(parser, MP_PARSE_NODE_NULL);
449+
push_result_node(&parser, MP_PARSE_NODE_NULL);
445450
backtrack = false;
446451
} else {
447452
// a mandatory rule that failed, so propagate backtrack
@@ -463,7 +468,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
463468
if (mp_lexer_is_kind(lex, tok_kind)) {
464469
// matched token
465470
if (tok_kind == MP_TOKEN_NAME) {
466-
push_result_token(parser, lex);
471+
push_result_token(&parser, lex);
467472
}
468473
mp_lexer_to_next(lex);
469474
} else {
@@ -480,8 +485,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
480485
break;
481486
case RULE_ARG_RULE:
482487
case RULE_ARG_OPT_RULE:
483-
push_rule(parser, rule_src_line, rule, i + 1); // save this and-rule
484-
push_rule_from_arg(parser, rule->arg[i]); // push child of and-rule
488+
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
489+
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
485490
goto next_rule;
486491
default:
487492
assert(0);
@@ -514,12 +519,12 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
514519
#if 0 && !MICROPY_ENABLE_DOC_STRING
515520
// this code discards lonely statement, such as doc strings
516521
// problem is that doc strings have already been interned, so this doesn't really help reduce RAM usage
517-
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(parser, 0) == MP_PARSE_NODE_NULL) {
518-
mp_parse_node_t p = peek_result(parser, 1);
522+
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
523+
mp_parse_node_t p = peek_result(&parser, 1);
519524
if (MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) {
520525
pop_result(parser);
521526
pop_result(parser);
522-
push_result_rule(parser, rule_src_line, rules[RULE_pass_stmt], 0);
527+
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
523528
break;
524529
}
525530
}
@@ -540,35 +545,35 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
540545
// always emit these rules, and add an extra blank node at the end (to be used by the compiler to store data)
541546
if (ADD_BLANK_NODE(rule->rule_id)) {
542547
emit_rule = true;
543-
push_result_node(parser, MP_PARSE_NODE_NULL);
548+
push_result_node(&parser, MP_PARSE_NODE_NULL);
544549
i += 1;
545550
}
546551

547552
int num_not_nil = 0;
548553
for (int x = 0; x < i; ++x) {
549-
if (peek_result(parser, x) != MP_PARSE_NODE_NULL) {
554+
if (peek_result(&parser, x) != MP_PARSE_NODE_NULL) {
550555
num_not_nil += 1;
551556
}
552557
}
553558
//printf("done and %s n=%d i=%d notnil=%d\n", rule->rule_name, n, i, num_not_nil);
554559
if (emit_rule) {
555-
push_result_rule(parser, rule_src_line, rule, i);
560+
push_result_rule(&parser, rule_src_line, rule, i);
556561
} else if (num_not_nil == 0) {
557-
push_result_rule(parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
562+
push_result_rule(&parser, rule_src_line, rule, i); // needed for, eg, atom_paren, testlist_comp_3b
558563
//result_stack_show(parser);
559564
//assert(0);
560565
} else if (num_not_nil == 1) {
561566
// single result, leave it on stack
562567
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
563568
for (int x = 0; x < i; ++x) {
564-
mp_parse_node_t pn2 = pop_result(parser);
569+
mp_parse_node_t pn2 = pop_result(&parser);
565570
if (pn2 != MP_PARSE_NODE_NULL) {
566571
pn = pn2;
567572
}
568573
}
569-
push_result_node(parser, pn);
574+
push_result_node(&parser, pn);
570575
} else {
571-
push_result_rule(parser, rule_src_line, rule, i);
576+
push_result_rule(&parser, rule_src_line, rule, i);
572577
}
573578
break;
574579

@@ -615,7 +620,7 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
615620
if (i & 1 & n) {
616621
// separators which are tokens are not pushed to result stack
617622
} else {
618-
push_result_token(parser, lex);
623+
push_result_token(&parser, lex);
619624
}
620625
mp_lexer_to_next(lex);
621626
// got element of list, so continue parsing list
@@ -628,8 +633,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
628633
}
629634
break;
630635
case RULE_ARG_RULE:
631-
push_rule(parser, rule_src_line, rule, i + 1); // save this list-rule
632-
push_rule_from_arg(parser, arg); // push child of list-rule
636+
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
637+
push_rule_from_arg(&parser, arg); // push child of list-rule
633638
goto next_rule;
634639
default:
635640
assert(0);
@@ -649,13 +654,13 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
649654
// list matched single item
650655
if (had_trailing_sep) {
651656
// if there was a trailing separator, make a list of a single item
652-
push_result_rule(parser, rule_src_line, rule, i);
657+
push_result_rule(&parser, rule_src_line, rule, i);
653658
} else {
654659
// just leave single item on stack (ie don't wrap in a list)
655660
}
656661
} else {
657662
//printf("done list %s %d %d\n", rule->rule_name, n, i);
658-
push_result_rule(parser, rule_src_line, rule, i);
663+
push_result_rule(&parser, rule_src_line, rule, i);
659664
}
660665
break;
661666

@@ -667,7 +672,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
667672
mp_parse_node_t result;
668673

669674
// check if we had a memory error
670-
if (parser->had_memory_error) {
675+
if (parser.had_memory_error) {
676+
memory_error:
671677
*parse_error_kind_out = MP_PARSE_ERROR_MEMORY;
672678
result = MP_PARSE_NODE_NULL;
673679
goto finished;
@@ -681,19 +687,18 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind, mp_p
681687

682688
//printf("--------------\n");
683689
//result_stack_show(parser);
684-
//printf("rule stack alloc: %d\n", parser->rule_stack_alloc);
685-
//printf("result stack alloc: %d\n", parser->result_stack_alloc);
690+
//printf("rule stack alloc: %d\n", parser.rule_stack_alloc);
691+
//printf("result stack alloc: %d\n", parser.result_stack_alloc);
686692
//printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
687693

688694
// get the root parse node that we created
689-
assert(parser->result_stack_top == 1);
690-
result = parser->result_stack[0];
695+
assert(parser.result_stack_top == 1);
696+
result = parser.result_stack[0];
691697

692698
finished:
693699
// free the memory that we don't need anymore
694-
m_del(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc);
695-
m_del(mp_parse_node_t, parser->result_stack, parser->result_stack_alloc);
696-
m_del_obj(parser_t, parser);
700+
m_del(rule_stack_t, parser.rule_stack, parser.rule_stack_alloc);
701+
m_del(mp_parse_node_t, parser.result_stack, parser.result_stack_alloc);
697702

698703
// return the result
699704
return result;

0 commit comments

Comments
 (0)