Skip to content

Commit ffa9bdd

Browse files
committed
Make lexerunix not allocate RAM for the entire script.
Now reads in small chunks at a time.
1 parent f0954e3 commit ffa9bdd

1 file changed

Lines changed: 36 additions & 14 deletions

File tree

py/lexerunix.c

Lines changed: 36 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,23 +13,45 @@
1313
#include <sys/stat.h>
1414
#include <sys/types.h>
1515

16-
mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
17-
int fd = open(filename, O_RDONLY);
18-
if (fd < 0) {
19-
return NULL;
16+
typedef struct _mp_lexer_file_buf_t {
17+
int fd;
18+
char buf[20];
19+
uint len;
20+
uint pos;
21+
} mp_lexer_file_buf_t;
22+
23+
STATIC unichar file_buf_next_char(mp_lexer_file_buf_t *fb) {
24+
if (fb->pos >= fb->len) {
25+
if (fb->len < sizeof(fb->buf)) {
26+
return MP_LEXER_CHAR_EOF;
27+
} else {
28+
int n = read(fb->fd, fb->buf, sizeof(fb->buf));
29+
if (n <= 0) {
30+
return MP_LEXER_CHAR_EOF;
31+
}
32+
fb->len = n;
33+
fb->pos = 0;
34+
}
2035
}
21-
uint size = lseek(fd, 0, SEEK_END);
22-
lseek(fd, 0, SEEK_SET);
23-
char *data = m_new(char, size);
24-
int read_size = read(fd, data, size);
25-
close(fd);
26-
if (read_size != size) {
27-
printf("error reading file %s\n", filename);
28-
m_del(char, data, size);
36+
return fb->buf[fb->pos++];
37+
}
38+
39+
STATIC void file_buf_close(mp_lexer_file_buf_t *fb) {
40+
close(fb->fd);
41+
m_del_obj(mp_lexer_file_buf_t, fb);
42+
}
43+
44+
mp_lexer_t *mp_lexer_new_from_file(const char *filename) {
45+
mp_lexer_file_buf_t *fb = m_new_obj(mp_lexer_file_buf_t);
46+
fb->fd = open(filename, O_RDONLY);
47+
if (fb->fd < 0) {
48+
m_del_obj(mp_lexer_file_buf_t, fb);
2949
return NULL;
3050
}
31-
32-
return mp_lexer_new_from_str_len(qstr_from_str(filename), data, size, size);
51+
int n = read(fb->fd, fb->buf, sizeof(fb->buf));
52+
fb->len = n;
53+
fb->pos = 0;
54+
return mp_lexer_new(qstr_from_str(filename), fb, (mp_lexer_stream_next_char_t)file_buf_next_char, (mp_lexer_stream_close_t)file_buf_close);
3355
}
3456

3557
#endif // MICROPY_ENABLE_LEXER_UNIX

0 commit comments

Comments
 (0)