Skip to content

Commit 8588525

Browse files
dpgeorgedpgeorge
committed
py/compile: De-duplicate constant objects in module's constant table.
The recent rework of bytecode made all constants global with respect to the module (previously, each function had its own constant table). That means the constant table for a module is shared among all functions/methods/etc within the module. This commit add support to the compiler to de-duplicate constants in this module constant table. So if a constant is used more than once -- eg 1.0 or (None, None) -- then the same object is reused for all instances. For example, if there is code like `print(1.0, 1.0)` then the parser will create two independent constants 1.0 and 1.0. The compiler will then (with this commit) notice they are the same and only put one of them in the constant table. The bytecode will then reuse that constant twice in the print expression. That allows the second 1.0 to be reclaimed by the GC, also means the constant table has one less entry so saves a word. Signed-off-by: Damien George <damien@micropython.org>
1 parent b3d0f5f commit 8588525

5 files changed

Lines changed: 76 additions & 52 deletions

File tree

‎py/compile.c‎

Lines changed: 19 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -185,8 +185,6 @@ typedef struct _compiler_t {
185185
scope_t *scope_head;
186186
scope_t *scope_cur;
187187

188-
mp_emit_common_t emit_common;
189-
190188
emit_t *emit; // current emitter
191189
#if NEED_METHOD_TABLE
192190
const emit_method_table_t *emit_method_table; // current emit method table
@@ -196,6 +194,8 @@ typedef struct _compiler_t {
196194
emit_inline_asm_t *emit_inline_asm; // current emitter for inline asm
197195
const emit_inline_asm_method_table_t *emit_inline_asm_method_table; // current emit method table for inline asm
198196
#endif
197+
198+
mp_emit_common_t emit_common;
199199
} compiler_t;
200200

201201
/******************************************************************************/
@@ -210,15 +210,11 @@ STATIC void mp_emit_common_init(mp_emit_common_t *emit, qstr source_file) {
210210
mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(source_file), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND);
211211
elem->value = MP_OBJ_NEW_SMALL_INT(0);
212212
#endif
213+
mp_obj_list_init(&emit->const_obj_list, 0);
213214
}
214215

215216
STATIC void mp_emit_common_start_pass(mp_emit_common_t *emit, pass_kind_t pass) {
216217
emit->pass = pass;
217-
if (pass == MP_PASS_STACK_SIZE) {
218-
emit->ct_cur_obj_base = emit->ct_cur_obj;
219-
} else if (pass > MP_PASS_STACK_SIZE) {
220-
emit->ct_cur_obj = emit->ct_cur_obj_base;
221-
}
222218
if (pass == MP_PASS_CODE_SIZE) {
223219
if (emit->ct_cur_child == 0) {
224220
emit->children = NULL;
@@ -229,22 +225,10 @@ STATIC void mp_emit_common_start_pass(mp_emit_common_t *emit, pass_kind_t pass)
229225
emit->ct_cur_child = 0;
230226
}
231227

232-
STATIC void mp_emit_common_finalise(mp_emit_common_t *emit, bool has_native_code) {
233-
emit->ct_cur_obj += has_native_code; // allocate an additional slot for &mp_fun_table
234-
emit->const_table = m_new0(mp_uint_t, emit->ct_cur_obj);
235-
emit->ct_cur_obj = has_native_code; // reserve slot 0 for &mp_fun_table
236-
#if MICROPY_EMIT_NATIVE
237-
if (has_native_code) {
238-
// store mp_fun_table pointer at the start of the constant table
239-
emit->const_table[0] = (mp_uint_t)(uintptr_t)&mp_fun_table;
240-
}
241-
#endif
242-
}
243-
244228
STATIC void mp_emit_common_populate_module_context(mp_emit_common_t *emit, qstr source_file, mp_module_context_t *context) {
245229
#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE
246230
size_t qstr_map_used = emit->qstr_map.used;
247-
mp_module_context_alloc_tables(context, qstr_map_used, emit->ct_cur_obj);
231+
mp_module_context_alloc_tables(context, qstr_map_used, emit->const_obj_list.len);
248232
for (size_t i = 0; i < emit->qstr_map.alloc; ++i) {
249233
if (mp_map_slot_is_filled(&emit->qstr_map, i)) {
250234
size_t idx = MP_OBJ_SMALL_INT_VALUE(emit->qstr_map.table[i].value);
@@ -253,12 +237,12 @@ STATIC void mp_emit_common_populate_module_context(mp_emit_common_t *emit, qstr
253237
}
254238
}
255239
#else
256-
mp_module_context_alloc_tables(context, 0, emit->ct_cur_obj);
240+
mp_module_context_alloc_tables(context, 0, emit->const_obj_list.len);
257241
context->constants.source_file = source_file;
258242
#endif
259243

260-
if (emit->ct_cur_obj > 0) {
261-
memcpy(context->constants.obj_table, emit->const_table, emit->ct_cur_obj * sizeof(mp_uint_t));
244+
for (size_t i = 0; i < emit->const_obj_list.len; ++i) {
245+
context->constants.obj_table[i] = emit->const_obj_list.items[i];
262246
}
263247
}
264248

@@ -3501,23 +3485,13 @@ mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr so
35013485
}
35023486

35033487
// compute some things related to scope and identifiers
3504-
bool has_native_code = false;
35053488
for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) {
3506-
#if MICROPY_EMIT_NATIVE
3507-
if (s->emit_options == MP_EMIT_OPT_NATIVE_PYTHON || s->emit_options == MP_EMIT_OPT_VIPER) {
3508-
has_native_code = true;
3509-
}
3510-
#endif
3511-
35123489
scope_compute_things(s);
35133490
}
35143491

35153492
// set max number of labels now that it's calculated
35163493
emit_bc_set_max_num_labels(emit_bc, max_num_labels);
35173494

3518-
// finalise and allocate the constant table
3519-
mp_emit_common_finalise(&comp->emit_common, has_native_code);
3520-
35213495
// compile MP_PASS_STACK_SIZE, MP_PASS_CODE_SIZE, MP_PASS_EMIT
35223496
#if MICROPY_EMIT_NATIVE
35233497
emit_t *emit_native = NULL;
@@ -3604,9 +3578,19 @@ mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr so
36043578
cm.rc = module_scope->raw_code;
36053579
cm.context = context;
36063580
#if MICROPY_PERSISTENT_CODE_SAVE
3607-
cm.has_native = has_native_code;
3581+
cm.has_native = false;
3582+
#if MICROPY_EMIT_NATIVE
3583+
if (emit_native != NULL) {
3584+
cm.has_native = true;
3585+
}
3586+
#endif
3587+
#if MICROPY_EMIT_INLINE_ASM
3588+
if (comp->emit_inline_asm != NULL) {
3589+
cm.has_native = true;
3590+
}
3591+
#endif
36083592
cm.n_qstr = comp->emit_common.qstr_map.used;
3609-
cm.n_obj = comp->emit_common.ct_cur_obj;
3593+
cm.n_obj = comp->emit_common.const_obj_list.len;
36103594
#endif
36113595
if (comp->compile_error == MP_OBJ_NULL) {
36123596
mp_emit_common_populate_module_context(&comp->emit_common, source_file, context);

‎py/emit.h‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -94,14 +94,12 @@ typedef struct _emit_t emit_t;
9494

9595
typedef struct _mp_emit_common_t {
9696
pass_kind_t pass;
97-
uint16_t ct_cur_obj_base;
98-
uint16_t ct_cur_obj;
9997
uint16_t ct_cur_child;
100-
mp_uint_t *const_table;
10198
mp_raw_code_t **children;
10299
#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE
103100
mp_map_t qstr_map;
104101
#endif
102+
mp_obj_list_t const_obj_list;
105103
} mp_emit_common_t;
106104

107105
typedef struct _mp_emit_method_table_id_ops_t {
@@ -181,12 +179,7 @@ static inline qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr
181179
}
182180
#endif
183181

184-
static inline size_t mp_emit_common_alloc_const_obj(mp_emit_common_t *emit, mp_obj_t obj) {
185-
if (emit->pass == MP_PASS_EMIT) {
186-
emit->const_table[emit->ct_cur_obj] = (mp_uint_t)obj;
187-
}
188-
return emit->ct_cur_obj++;
189-
}
182+
size_t mp_emit_common_use_const_obj(mp_emit_common_t *emit, mp_obj_t const_obj);
190183

191184
static inline size_t mp_emit_common_alloc_const_child(mp_emit_common_t *emit, mp_raw_code_t *rc) {
192185
if (emit->pass == MP_PASS_EMIT) {

‎py/emitbc.c‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -204,8 +204,7 @@ STATIC void emit_write_bytecode_byte_qstr(emit_t *emit, int stack_adj, byte b, q
204204
}
205205

206206
STATIC void emit_write_bytecode_byte_obj(emit_t *emit, int stack_adj, byte b, mp_obj_t obj) {
207-
emit_write_bytecode_byte_const(emit, stack_adj, b,
208-
mp_emit_common_alloc_const_obj(emit->emit_common, obj));
207+
emit_write_bytecode_byte_const(emit, stack_adj, b, mp_emit_common_use_const_obj(emit->emit_common, obj));
209208
}
210209

211210
STATIC void emit_write_bytecode_byte_child(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) {

‎py/emitcommon.c‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include <assert.h>
2828

2929
#include "py/emit.h"
30+
#include "py/nativeglue.h"
3031

3132
#if MICROPY_ENABLE_COMPILER
3233

@@ -40,6 +41,51 @@ qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) {
4041
}
4142
#endif
4243

44+
// Compare two objects for strict equality, including equality of type. This is
45+
// different to the semantics of mp_obj_equal which, eg, has (True,) == (1.0,).
46+
static bool strictly_equal(mp_obj_t a, mp_obj_t b) {
47+
if (a == b) {
48+
return true;
49+
}
50+
51+
#if MICROPY_EMIT_NATIVE
52+
if (a == MP_OBJ_FROM_PTR(&mp_fun_table) || b == MP_OBJ_FROM_PTR(&mp_fun_table)) {
53+
return false;
54+
}
55+
#endif
56+
57+
const mp_obj_type_t *a_type = mp_obj_get_type(a);
58+
const mp_obj_type_t *b_type = mp_obj_get_type(b);
59+
if (a_type != b_type) {
60+
return false;
61+
}
62+
if (a_type == &mp_type_tuple) {
63+
mp_obj_tuple_t *a_tuple = MP_OBJ_TO_PTR(a);
64+
mp_obj_tuple_t *b_tuple = MP_OBJ_TO_PTR(b);
65+
if (a_tuple->len != b_tuple->len) {
66+
return false;
67+
}
68+
for (size_t i = 0; i < a_tuple->len; ++i) {
69+
if (!strictly_equal(a_tuple->items[i], b_tuple->items[i])) {
70+
return false;
71+
}
72+
}
73+
return true;
74+
} else {
75+
return mp_obj_equal(a, b);
76+
}
77+
}
78+
79+
size_t mp_emit_common_use_const_obj(mp_emit_common_t *emit, mp_obj_t const_obj) {
80+
for (size_t i = 0; i < emit->const_obj_list.len; ++i) {
81+
if (strictly_equal(emit->const_obj_list.items[i], const_obj)) {
82+
return i;
83+
}
84+
}
85+
mp_obj_list_append(MP_OBJ_FROM_PTR(&emit->const_obj_list), const_obj);
86+
return emit->const_obj_list.len - 1;
87+
}
88+
4389
void mp_emit_common_get_id_for_modification(scope_t *scope, qstr qst) {
4490
// name adding/lookup
4591
id_info_t *id = scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT);

‎py/emitnative.c‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,6 @@
9999
#define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t))
100100
#define OFFSETOF_MODULE_CONTEXT_OBJ_TABLE (offsetof(mp_module_context_t, constants.obj_table) / sizeof(uintptr_t))
101101
#define OFFSETOF_MODULE_CONTEXT_GLOBALS (offsetof(mp_module_context_t, module.globals) / sizeof(uintptr_t))
102-
#define INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE (0)
103102

104103
// If not already defined, set parent args to same as child call registers
105104
#ifndef REG_PARENT_RET
@@ -406,6 +405,8 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
406405
emit->code_state_start = SIZEOF_NLR_BUF;
407406
}
408407

408+
size_t fun_table_off = mp_emit_common_use_const_obj(emit->emit_common, MP_OBJ_FROM_PTR(&mp_fun_table));
409+
409410
if (emit->do_viper_types) {
410411
// Work out size of state (locals plus stack)
411412
// n_state counts all stack and locals, even those in registers
@@ -443,7 +444,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
443444
// Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
444445
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT);
445446
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE);
446-
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE);
447+
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, fun_table_off);
447448

448449
// Store function object (passed as first arg) to stack if needed
449450
if (NEED_FUN_OBJ(emit)) {
@@ -520,7 +521,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
520521
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, LOCAL_IDX_FUN_OBJ(emit));
521522
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT);
522523
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE);
523-
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE);
524+
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, fun_table_off);
524525
} else {
525526
// The locals and stack start after the code_state structure
526527
emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE;
@@ -540,7 +541,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop
540541
// Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table
541542
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT);
542543
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE);
543-
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE);
544+
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, fun_table_off);
544545

545546
// Set code_state.fun_bc
546547
ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1);
@@ -1105,7 +1106,7 @@ STATIC exc_stack_entry_t *emit_native_pop_exc_stack(emit_t *emit) {
11051106
}
11061107

11071108
STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) {
1108-
size_t table_off = mp_emit_common_alloc_const_obj(emit->emit_common, obj);
1109+
size_t table_off = mp_emit_common_use_const_obj(emit->emit_common, obj);
11091110
emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit));
11101111
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT);
11111112
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE);
@@ -1211,10 +1212,11 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) {
12111212
emit_native_label_assign(emit, global_except_label);
12121213
#if N_NLR_SETJMP
12131214
// Reload REG_FUN_TABLE, since it may be clobbered by longjmp
1215+
size_t fun_table_off = mp_emit_common_use_const_obj(emit->emit_common, MP_OBJ_FROM_PTR(&mp_fun_table));
12141216
emit_native_mov_reg_state(emit, REG_LOCAL_1, LOCAL_IDX_FUN_OBJ(emit));
12151217
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, OFFSETOF_OBJ_FUN_BC_CONTEXT);
12161218
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE);
1217-
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, emit->scope->num_pos_args + emit->scope->num_kwonly_args);
1219+
ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, fun_table_off);
12181220
#endif
12191221
ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, LOCAL_IDX_EXC_HANDLER_PC(emit));
12201222
ASM_JUMP_IF_REG_NONZERO(emit->as, REG_LOCAL_1, nlr_label, false);

0 commit comments

Comments
 (0)