Skip to content

Commit 1f54ad2

Browse files
dpgeorgedpgeorge
committed
py: Make interning of qstrs thread safe.
1 parent b7274e9 commit 1f54ad2

2 files changed

Lines changed: 36 additions & 2 deletions

File tree

‎py/mpstate.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,11 @@ typedef struct _mp_state_vm_t {
164164
size_t qstr_last_alloc;
165165
size_t qstr_last_used;
166166

167+
#if MICROPY_PY_THREAD
168+
// This is a global mutex used to make qstr interning thread-safe.
169+
mp_thread_mutex_t qstr_mutex;
170+
#endif
171+
167172
mp_uint_t mp_optimise_value;
168173

169174
// size of the emergency exception buf, if it's dynamically allocated

‎py/qstr.c‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,14 @@
7272
#error unimplemented qstr length decoding
7373
#endif
7474

75+
#if MICROPY_PY_THREAD
76+
#define QSTR_ENTER() mp_thread_mutex_lock(&MP_STATE_VM(qstr_mutex), 1)
77+
#define QSTR_EXIT() mp_thread_mutex_unlock(&MP_STATE_VM(qstr_mutex))
78+
#else
79+
#define QSTR_ENTER()
80+
#define QSTR_EXIT()
81+
#endif
82+
7583
// this must match the equivalent function in makeqstrdata.py
7684
mp_uint_t qstr_compute_hash(const byte *data, size_t len) {
7785
// djb2 algorithm; see http://www.cse.yorku.ca/~oz/hash.html
@@ -111,6 +119,10 @@ extern const qstr_pool_t MICROPY_QSTR_EXTRA_POOL;
111119
void qstr_init(void) {
112120
MP_STATE_VM(last_pool) = (qstr_pool_t*)&CONST_POOL; // we won't modify the const_pool since it has no allocated room left
113121
MP_STATE_VM(qstr_last_chunk) = NULL;
122+
123+
#if MICROPY_PY_THREAD
124+
mp_thread_mutex_init(&MP_STATE_VM(qstr_mutex));
125+
#endif
114126
}
115127

116128
STATIC const byte *find_qstr(qstr q) {
@@ -125,12 +137,17 @@ STATIC const byte *find_qstr(qstr q) {
125137
return 0;
126138
}
127139

140+
// qstr_mutex must be taken while in this function
128141
STATIC qstr qstr_add(const byte *q_ptr) {
129142
DEBUG_printf("QSTR: add hash=%d len=%d data=%.*s\n", Q_GET_HASH(q_ptr), Q_GET_LENGTH(q_ptr), Q_GET_LENGTH(q_ptr), Q_GET_DATA(q_ptr));
130143

131144
// make sure we have room in the pool for a new qstr
132145
if (MP_STATE_VM(last_pool)->len >= MP_STATE_VM(last_pool)->alloc) {
133-
qstr_pool_t *pool = m_new_obj_var(qstr_pool_t, const char*, MP_STATE_VM(last_pool)->alloc * 2);
146+
qstr_pool_t *pool = m_new_obj_var_maybe(qstr_pool_t, const char*, MP_STATE_VM(last_pool)->alloc * 2);
147+
if (pool == NULL) {
148+
QSTR_EXIT();
149+
m_malloc_fail(MP_STATE_VM(last_pool)->alloc * 2);
150+
}
134151
pool->prev = MP_STATE_VM(last_pool);
135152
pool->total_prev_len = MP_STATE_VM(last_pool)->total_prev_len + MP_STATE_VM(last_pool)->len;
136153
pool->alloc = MP_STATE_VM(last_pool)->alloc * 2;
@@ -169,6 +186,7 @@ qstr qstr_from_str(const char *str) {
169186

170187
qstr qstr_from_strn(const char *str, size_t len) {
171188
assert(len < (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN)));
189+
QSTR_ENTER();
172190
qstr q = qstr_find_strn(str, len);
173191
if (q == 0) {
174192
// qstr does not exist in interned pool so need to add it
@@ -198,7 +216,11 @@ qstr qstr_from_strn(const char *str, size_t len) {
198216
MP_STATE_VM(qstr_last_chunk) = m_new_maybe(byte, al);
199217
if (MP_STATE_VM(qstr_last_chunk) == NULL) {
200218
// failed to allocate a large chunk so try with exact size
201-
MP_STATE_VM(qstr_last_chunk) = m_new(byte, n_bytes);
219+
MP_STATE_VM(qstr_last_chunk) = m_new_maybe(byte, n_bytes);
220+
if (MP_STATE_VM(qstr_last_chunk) == NULL) {
221+
QSTR_EXIT();
222+
m_malloc_fail(n_bytes);
223+
}
202224
al = n_bytes;
203225
}
204226
MP_STATE_VM(qstr_last_alloc) = al;
@@ -217,6 +239,7 @@ qstr qstr_from_strn(const char *str, size_t len) {
217239
q_ptr[MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len] = '\0';
218240
q = qstr_add(q_ptr);
219241
}
242+
QSTR_EXIT();
220243
return q;
221244
}
222245

@@ -228,6 +251,7 @@ byte *qstr_build_start(size_t len, byte **q_ptr) {
228251
}
229252

230253
qstr qstr_build_end(byte *q_ptr) {
254+
QSTR_ENTER();
231255
qstr q = qstr_find_strn((const char*)Q_GET_DATA(q_ptr), Q_GET_LENGTH(q_ptr));
232256
if (q == 0) {
233257
size_t len = Q_GET_LENGTH(q_ptr);
@@ -238,6 +262,7 @@ qstr qstr_build_end(byte *q_ptr) {
238262
} else {
239263
m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr));
240264
}
265+
QSTR_EXIT();
241266
return q;
242267
}
243268

@@ -263,6 +288,7 @@ const byte *qstr_data(qstr q, size_t *len) {
263288
}
264289

265290
void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, size_t *n_total_bytes) {
291+
QSTR_ENTER();
266292
*n_pool = 0;
267293
*n_qstr = 0;
268294
*n_str_data_bytes = 0;
@@ -280,14 +306,17 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
280306
#endif
281307
}
282308
*n_total_bytes += *n_str_data_bytes;
309+
QSTR_EXIT();
283310
}
284311

285312
#if MICROPY_PY_MICROPYTHON_MEM_INFO
286313
void qstr_dump_data(void) {
314+
QSTR_ENTER();
287315
for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
288316
for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
289317
mp_printf(&mp_plat_print, "Q(%s)\n", Q_GET_DATA(*q));
290318
}
291319
}
320+
QSTR_EXIT();
292321
}
293322
#endif

0 commit comments

Comments
 (0)