Skip to content

Commit 534b7c3

Browse files
dpgeorgedpgeorge
committed
py: Do adjacent str/bytes literal concatenation in lexer, not compiler.
It's much more efficient in RAM and code size to do implicit literal string concatenation in the lexer, as opposed to the compiler. RAM usage is reduced because the concatenation can be done right away in the tokeniser by just accumulating the string/bytes literals into the lexer's vstr. Prior to this patch adjacent strings/bytes would create a parse tree (one node per string/bytes) and then in the compiler a whole new chunk of memory was allocated to store the concatenated string, which used more than double the memory compared to just accumulating in the lexer. This patch also significantly reduces code size: bare-arm: -204 minimal: -204 unix x64: -328 stmhal: -208 esp8266: -284 cc3200: -224
1 parent 773278e commit 534b7c3

3 files changed

Lines changed: 199 additions & 218 deletions

File tree

‎py/compile.c‎

Lines changed: 0 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -2301,65 +2301,6 @@ STATIC void compile_atom_expr_trailers(compiler_t *comp, mp_parse_node_struct_t
23012301
}
23022302
}
23032303

2304-
STATIC void compile_atom_string(compiler_t *comp, mp_parse_node_struct_t *pns) {
2305-
// a list of strings
2306-
2307-
// check type of list (string or bytes) and count total number of bytes
2308-
int n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
2309-
size_t n_bytes = 0;
2310-
int string_kind = MP_PARSE_NODE_NULL;
2311-
for (int i = 0; i < n; i++) {
2312-
int pn_kind;
2313-
if (MP_PARSE_NODE_IS_LEAF(pns->nodes[i])) {
2314-
pn_kind = MP_PARSE_NODE_LEAF_KIND(pns->nodes[i]);
2315-
assert(pn_kind == MP_PARSE_NODE_STRING || pn_kind == MP_PARSE_NODE_BYTES);
2316-
n_bytes += qstr_len(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]));
2317-
} else {
2318-
assert(MP_PARSE_NODE_IS_STRUCT(pns->nodes[i]));
2319-
mp_parse_node_struct_t *pns_string = (mp_parse_node_struct_t*)pns->nodes[i];
2320-
if (MP_PARSE_NODE_STRUCT_KIND(pns_string) == PN_string) {
2321-
pn_kind = MP_PARSE_NODE_STRING;
2322-
} else {
2323-
assert(MP_PARSE_NODE_STRUCT_KIND(pns_string) == PN_bytes);
2324-
pn_kind = MP_PARSE_NODE_BYTES;
2325-
}
2326-
n_bytes += pns_string->nodes[1];
2327-
}
2328-
if (i == 0) {
2329-
string_kind = pn_kind;
2330-
} else if (pn_kind != string_kind) {
2331-
compile_syntax_error(comp, (mp_parse_node_t)pns, "cannot mix bytes and nonbytes literals");
2332-
return;
2333-
}
2334-
}
2335-
2336-
// if we are not in the last pass, just load a dummy object
2337-
if (comp->pass != MP_PASS_EMIT) {
2338-
EMIT_ARG(load_const_obj, mp_const_none);
2339-
return;
2340-
}
2341-
2342-
// concatenate string/bytes
2343-
vstr_t vstr;
2344-
vstr_init_len(&vstr, n_bytes);
2345-
byte *s_dest = (byte*)vstr.buf;
2346-
for (int i = 0; i < n; i++) {
2347-
if (MP_PARSE_NODE_IS_LEAF(pns->nodes[i])) {
2348-
size_t s_len;
2349-
const byte *s = qstr_data(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i]), &s_len);
2350-
memcpy(s_dest, s, s_len);
2351-
s_dest += s_len;
2352-
} else {
2353-
mp_parse_node_struct_t *pns_string = (mp_parse_node_struct_t*)pns->nodes[i];
2354-
memcpy(s_dest, (const char*)pns_string->nodes[0], pns_string->nodes[1]);
2355-
s_dest += pns_string->nodes[1];
2356-
}
2357-
}
2358-
2359-
// load the object
2360-
EMIT_ARG(load_const_obj, mp_obj_new_str_from_vstr(string_kind == MP_PARSE_NODE_STRING ? &mp_type_str : &mp_type_bytes, &vstr));
2361-
}
2362-
23632304
// pns needs to have 2 nodes, first is lhs of comprehension, second is PN_comp_for node
23642305
STATIC void compile_comprehension(compiler_t *comp, mp_parse_node_struct_t *pns, scope_kind_t kind) {
23652306
assert(MP_PARSE_NODE_STRUCT_NUM_NODES(pns) == 2);

‎py/grammar.h‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -268,8 +268,7 @@ DEF_RULE_NC(power_dbl_star, and_ident(2), tok(OP_DBL_STAR), rule(factor))
268268
// testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
269269
// trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
270270

271-
DEF_RULE_NC(atom, or(11), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), rule(atom_string), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace))
272-
DEF_RULE(atom_string, c(atom_string), one_or_more, rule(string_or_bytes))
271+
DEF_RULE_NC(atom, or(12), tok(NAME), tok(INTEGER), tok(FLOAT_OR_IMAG), tok(STRING), tok(BYTES), tok(ELLIPSIS), tok(KW_NONE), tok(KW_TRUE), tok(KW_FALSE), rule(atom_paren), rule(atom_bracket), rule(atom_brace))
273272
DEF_RULE_NC(string_or_bytes, or(2), tok(STRING), tok(BYTES))
274273
DEF_RULE(atom_paren, c(atom_paren), and(3), tok(DEL_PAREN_OPEN), opt_rule(atom_2b), tok(DEL_PAREN_CLOSE))
275274
DEF_RULE_NC(atom_2b, or(2), rule(yield_expr), rule(testlist_comp))

0 commit comments

Comments
 (0)