Skip to content

Commit fa2162b

Browse files
dpgeorgedpgeorge
committed
Integrate new lexer stream with stm framework.
1 parent 27fb45e commit fa2162b

10 files changed

Lines changed: 126 additions & 31 deletions

File tree

‎py/lexer.c‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ struct _py_lexer_t {
1414
const char *name; // name of source
1515
void *stream_data; // data for stream
1616
py_lexer_stream_next_char_t stream_next_char; // stream callback to get next char
17-
py_lexer_stream_free_t stream_free; // stream callback to free
17+
py_lexer_stream_close_t stream_close; // stream callback to free
1818

1919
unichar chr0, chr1, chr2; // current cached characters from source
2020

@@ -589,13 +589,13 @@ static void py_lexer_next_token_into(py_lexer_t *lex, py_token_t *tok, bool firs
589589
}
590590
}
591591

592-
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_free_t stream_free) {
592+
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_close_t stream_close) {
593593
py_lexer_t *lex = m_new(py_lexer_t, 1);
594594

595595
lex->name = src_name; // TODO do we need to strdup this?
596596
lex->stream_data = stream_data;
597597
lex->stream_next_char = stream_next_char;
598-
lex->stream_free = stream_free;
598+
lex->stream_close = stream_close;
599599
lex->line = 1;
600600
lex->column = 1;
601601
lex->emit_dent = 0;
@@ -632,8 +632,8 @@ py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_strea
632632

633633
void py_lexer_free(py_lexer_t *lex) {
634634
if (lex) {
635-
if (lex->stream_free) {
636-
lex->stream_free(lex->stream_data);
635+
if (lex->stream_close) {
636+
lex->stream_close(lex->stream_data);
637637
}
638638
m_free(lex);
639639
}

‎py/lexer.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -122,15 +122,15 @@ typedef struct _py_token_t {
122122
// it can be called again after returning PY_LEXER_CHAR_EOF, and in that case must return PY_LEXER_CHAR_EOF
123123
#define PY_LEXER_CHAR_EOF (-1)
124124
typedef unichar (*py_lexer_stream_next_char_t)(void*);
125-
typedef void (*py_lexer_stream_free_t)(void*);
125+
typedef void (*py_lexer_stream_close_t)(void*);
126126

127127
typedef struct _py_lexer_t py_lexer_t;
128128

129129
void py_token_show(const py_token_t *tok);
130130
void py_token_show_error_prefix(const py_token_t *tok);
131131
bool py_token_show_error(const py_token_t *tok, const char *msg);
132132

133-
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_free_t stream_free);
133+
py_lexer_t *py_lexer_new(const char *src_name, void *stream_data, py_lexer_stream_next_char_t stream_next_char, py_lexer_stream_close_t stream_close);
134134
void py_lexer_free(py_lexer_t *lex);
135135
void py_lexer_to_next(py_lexer_t *lex);
136136
const py_token_t *py_lexer_cur(const py_lexer_t *lex);

‎stm/Makefile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ SRC_C = \
2424
systick.c \
2525
stm32fxxx_it.c \
2626
usb.c \
27+
lexerstm.c \
2728
# sd.c \
2829
2930
SRC_S = \
@@ -33,6 +34,7 @@ PY_O = \
3334
nlrthumb.o \
3435
malloc.o \
3536
qstr.o \
37+
vstr.o \
3638
misc.o \
3739
lexer.o \
3840
parse.o \

‎stm/lexerstm.c‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
#include <stdint.h>
2+
#include <stdio.h>
3+
4+
#include "ff.h"
5+
6+
#include "misc.h"
7+
#include "lexer.h"
8+
#include "lexerstm.h"
9+
10+
unichar str_buf_next_char(py_lexer_str_buf_t *sb) {
11+
if (sb->src_cur < sb->src_end) {
12+
return *sb->src_cur++;
13+
} else {
14+
return PY_LEXER_CHAR_EOF;
15+
}
16+
}
17+
18+
void str_buf_free(py_lexer_str_buf_t *sb) {
19+
if (sb->free) {
20+
m_free((char*)sb->src_beg);
21+
}
22+
}
23+
24+
py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, py_lexer_str_buf_t *sb) {
25+
sb->free = free_str;
26+
sb->src_beg = str;
27+
sb->src_cur = str;
28+
sb->src_end = str + len;
29+
return py_lexer_new(src_name, sb, (py_lexer_stream_next_char_t)str_buf_next_char, (py_lexer_stream_close_t)str_buf_free);
30+
}
31+
32+
unichar file_buf_next_char(py_lexer_file_buf_t *fb) {
33+
if (fb->pos >= fb->len) {
34+
if (fb->len < sizeof(fb->buf)) {
35+
return PY_LEXER_CHAR_EOF;
36+
} else {
37+
UINT n;
38+
f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
39+
fb->len = n;
40+
fb->pos = 0;
41+
}
42+
}
43+
return fb->buf[fb->pos++];
44+
}
45+
46+
void file_buf_close(py_lexer_file_buf_t *fb) {
47+
f_close(&fb->fp);
48+
}
49+
50+
py_lexer_t *py_lexer_new_from_file(const char *filename, py_lexer_file_buf_t *fb) {
51+
FRESULT res = f_open(&fb->fp, filename, FA_READ);
52+
if (res != FR_OK) {
53+
return NULL;
54+
}
55+
UINT n;
56+
f_read(&fb->fp, fb->buf, sizeof(fb->buf), &n);
57+
fb->len = n;
58+
fb->pos = 0;
59+
return py_lexer_new(filename, fb, (py_lexer_stream_next_char_t)file_buf_next_char, (py_lexer_stream_close_t)file_buf_close);
60+
}

‎stm/lexerstm.h‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
typedef struct _py_lexer_str_buf_t {
2+
bool free; // free src_beg when done
3+
const char *src_beg; // beginning of source
4+
const char *src_cur; // current location in source
5+
const char *src_end; // end (exclusive) of source
6+
} py_lexer_str_buf_t;
7+
8+
typedef struct _py_lexer_file_buf_t {
9+
FIL fp;
10+
char buf[20];
11+
uint16_t len;
12+
uint16_t pos;
13+
} py_lexer_file_buf_t;
14+
15+
py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str, py_lexer_str_buf_t *sb);
16+
py_lexer_t *py_lexer_new_from_file(const char *filename, py_lexer_file_buf_t *fb);

‎stm/main.c‎

Lines changed: 19 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#include "storage.h"
1212
#include "mma.h"
1313
#include "usb.h"
14+
#include "ff.h"
1415

1516
static void impl02_c_version() {
1617
int x = 0;
@@ -63,8 +64,10 @@ void __fatal_error(const char *msg) {
6364
}
6465
}
6566

67+
#include "nlr.h"
6668
#include "misc.h"
6769
#include "lexer.h"
70+
#include "lexerstm.h"
6871
#include "mpyconfig.h"
6972
#include "parse.h"
7073
#include "compile.h"
@@ -88,10 +91,8 @@ py_obj_t pyb_sw() {
8891
}
8992
}
9093

91-
#include "ff.h"
9294
FATFS fatfs0;
9395

94-
#include "nlr.h"
9596

9697
/*
9798
void g(uint i) {
@@ -293,7 +294,7 @@ int main() {
293294
//sys_tick_delay_ms(1000);
294295

295296
// Python!
296-
if (0) {
297+
if (1) {
297298
//const char *pysrc = "def f():\n x=x+1\nprint(42)\n";
298299
const char *pysrc =
299300
// impl01.py
@@ -323,19 +324,17 @@ int main() {
323324
" x = x + 1\n"
324325
"f()\n";
325326
*/
326-
/*
327327
"print('in python!')\n"
328328
"x = 0\n"
329329
"while x < 4:\n"
330330
" pyb_led(True)\n"
331331
" pyb_delay(201)\n"
332332
" pyb_led(False)\n"
333333
" pyb_delay(201)\n"
334-
" x = x + 1\n"
334+
" x += 1\n"
335335
"print('press me!')\n"
336336
"while True:\n"
337337
" pyb_led(pyb_sw())\n";
338-
*/
339338
/*
340339
// impl16.py
341340
"@micropython.asm_thumb\n"
@@ -372,6 +371,7 @@ int main() {
372371
"except:\n"
373372
" print(x)\n";
374373
*/
374+
/*
375375
// impl19.py
376376
"# for loop\n"
377377
"def f():\n"
@@ -380,29 +380,27 @@ int main() {
380380
" for z in range(400):\n"
381381
" pass\n"
382382
"f()\n";
383+
*/
383384

384-
py_lexer_t *lex = py_lexer_from_str_len("<>", pysrc, strlen(pysrc), false);
385+
py_lexer_str_buf_t py_lexer_str_buf;
386+
py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", pysrc, strlen(pysrc), false, &py_lexer_str_buf);
385387

386-
if (0) {
387-
while (!py_lexer_is_kind(lex, PY_TOKEN_END)) {
388-
py_token_show(py_lexer_cur(lex));
389-
py_lexer_to_next(lex);
390-
sys_tick_delay_ms(1000);
391-
}
392-
} else {
393-
// nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
394-
printf("lex; al=%u\n", m_get_total_bytes_allocated());
395-
sys_tick_delay_ms(1000);
396-
py_parse_node_t pn = py_parse(lex, 0);
397-
//printf("----------------\n");
388+
// nalloc=1740;6340;6836 -> 140;4600;496 bytes for lexer, parser, compiler
389+
printf("lex; al=%u\n", m_get_total_bytes_allocated());
390+
sys_tick_delay_ms(1000);
391+
py_parse_node_t pn = py_parse(lex, PY_PARSE_FILE_INPUT);
392+
py_lexer_free(lex);
393+
if (pn != PY_PARSE_NODE_NULL) {
398394
printf("pars;al=%u\n", m_get_total_bytes_allocated());
399395
sys_tick_delay_ms(1000);
400396
//parse_node_show(pn, 0);
401-
py_compile(pn, false);
397+
bool comp_ok = py_compile(pn, false);
402398
printf("comp;al=%u\n", m_get_total_bytes_allocated());
403399
sys_tick_delay_ms(1000);
404400

405-
if (1) {
401+
if (!comp_ok) {
402+
printf("compile error\n");
403+
} else {
406404
// execute it!
407405

408406
// add some functions to the python namespace

‎stm/std.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ void *calloc(size_t sz, size_t n);
88
void *realloc(void *ptr, size_t n);
99

1010
void *memcpy(void *dest, const void *src, size_t n);
11+
void *memmove(void *dest, const void *src, size_t n);
1112
void *memset(void *s, int c, size_t n);
1213

1314
int strlen(const char *str);

‎stm/string0.c‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,21 @@ void *memcpy(void *dest, const void *src, size_t n) {
1111
return dest;
1212
}
1313

14+
void *memmove(void *dest, const void *src, size_t n) {
15+
if (src < dest && dest < src + n) {
16+
// need to copy backwards
17+
uint8_t *d = dest + n - 1;
18+
const uint8_t *s = src + n - 1;
19+
for (; n > 0; n--) {
20+
*d-- = *s--;
21+
}
22+
return dest;
23+
} else {
24+
// can use normal memcpy
25+
return memcpy(dest, src, n);
26+
}
27+
}
28+
1429
void *memset(void *s, int c, size_t n) {
1530
uint8_t *s2 = s;
1631
for (; n > 0; n--) {

‎unix/main.c‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,11 @@ void do_repl() {
6868
line = line3;
6969
}
7070
}
71+
7172
py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", line, strlen(line), false);
7273
py_parse_node_t pn = py_parse(lex, PY_PARSE_SINGLE_INPUT);
74+
py_lexer_free(lex);
75+
7376
if (pn != PY_PARSE_NODE_NULL) {
7477
//py_parse_node_show(pn, 0);
7578
bool comp_ok = py_compile(pn, true);
@@ -111,15 +114,15 @@ void do_file(const char *file) {
111114
// compile
112115

113116
py_parse_node_t pn = py_parse(lex, PY_PARSE_FILE_INPUT);
117+
py_lexer_free(lex);
118+
114119
if (pn != PY_PARSE_NODE_NULL) {
115120
//printf("----------------\n");
116121
//parse_node_show(pn, 0);
117122
//printf("----------------\n");
118123
bool comp_ok = py_compile(pn, false);
119124
//printf("----------------\n");
120125

121-
py_lexer_free(lex);
122-
123126
#if MICROPY_EMIT_CPYTHON
124127
if (!comp_ok) {
125128
printf("compile error\n");

‎unix/mpyconfig.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
// options to control how Micro Python is built
22

33
#define MICROPY_ENABLE_FLOAT (1)
4-
#define MICROPY_EMIT_CPYTHON (1)
4+
#define MICROPY_EMIT_CPYTHON (0)
55
#define MICROPY_EMIT_X64 (1)
66
#define MICROPY_EMIT_THUMB (0)
77
#define MICROPY_EMIT_INLINE_THUMB (0)

0 commit comments

Comments
 (0)