Skip to content

Commit a91f414

Browse files
dpgeorgedpgeorge
committed
py, lexer: Fix parsing of raw strings (allow escaping of quote).
1 parent f22626e commit a91f414

1 file changed

Lines changed: 44 additions & 39 deletions

File tree

‎py/lexer.c‎

Lines changed: 44 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -455,50 +455,55 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
455455
vstr_add_char(&lex->vstr, CUR_CHAR(lex));
456456
} else {
457457
n_closing = 0;
458-
if (!is_raw && is_char(lex, '\\')) {
458+
if (is_char(lex, '\\')) {
459459
next_char(lex);
460460
unichar c = CUR_CHAR(lex);
461-
switch (c) {
462-
case MP_LEXER_CHAR_EOF: break; // TODO a proper error message?
463-
case '\n': c = MP_LEXER_CHAR_EOF; break; // TODO check this works correctly (we are supposed to ignore it
464-
case '\\': break;
465-
case '\'': break;
466-
case '"': break;
467-
case 'a': c = 0x07; break;
468-
case 'b': c = 0x08; break;
469-
case 't': c = 0x09; break;
470-
case 'n': c = 0x0a; break;
471-
case 'v': c = 0x0b; break;
472-
case 'f': c = 0x0c; break;
473-
case 'r': c = 0x0d; break;
474-
case 'x':
475-
{
476-
uint num = 0;
477-
if (!get_hex(lex, 2, &num)) {
478-
// TODO error message
479-
assert(0);
480-
}
481-
c = num;
482-
break;
483-
}
484-
case 'N': break; // TODO \N{name} only in strings
485-
case 'u': break; // TODO \uxxxx only in strings
486-
case 'U': break; // TODO \Uxxxxxxxx only in strings
487-
default:
488-
if (c >= '0' && c <= '7') {
489-
// Octal sequence, 1-3 chars
490-
int digits = 3;
491-
int num = c - '0';
492-
while (is_following_odigit(lex) && --digits != 0) {
493-
next_char(lex);
494-
num = num * 8 + (CUR_CHAR(lex) - '0');
461+
if (is_raw) {
462+
// raw strings allow escaping of quotes, but the backslash is also emitted
463+
vstr_add_char(&lex->vstr, '\\');
464+
} else {
465+
switch (c) {
466+
case MP_LEXER_CHAR_EOF: break; // TODO a proper error message?
467+
case '\n': c = MP_LEXER_CHAR_EOF; break; // TODO check this works correctly (we are supposed to ignore it
468+
case '\\': break;
469+
case '\'': break;
470+
case '"': break;
471+
case 'a': c = 0x07; break;
472+
case 'b': c = 0x08; break;
473+
case 't': c = 0x09; break;
474+
case 'n': c = 0x0a; break;
475+
case 'v': c = 0x0b; break;
476+
case 'f': c = 0x0c; break;
477+
case 'r': c = 0x0d; break;
478+
case 'x':
479+
{
480+
uint num = 0;
481+
if (!get_hex(lex, 2, &num)) {
482+
// TODO error message
483+
assert(0);
495484
}
496485
c = num;
497-
} else {
498-
// unrecognised escape character; CPython lets this through verbatim as '\' and then the character
499-
vstr_add_char(&lex->vstr, '\\');
486+
break;
500487
}
501-
break;
488+
case 'N': break; // TODO \N{name} only in strings
489+
case 'u': break; // TODO \uxxxx only in strings
490+
case 'U': break; // TODO \Uxxxxxxxx only in strings
491+
default:
492+
if (c >= '0' && c <= '7') {
493+
// Octal sequence, 1-3 chars
494+
int digits = 3;
495+
int num = c - '0';
496+
while (is_following_odigit(lex) && --digits != 0) {
497+
next_char(lex);
498+
num = num * 8 + (CUR_CHAR(lex) - '0');
499+
}
500+
c = num;
501+
} else {
502+
// unrecognised escape character; CPython lets this through verbatim as '\' and then the character
503+
vstr_add_char(&lex->vstr, '\\');
504+
}
505+
break;
506+
}
502507
}
503508
if (c != MP_LEXER_CHAR_EOF) {
504509
vstr_add_char(&lex->vstr, c);

0 commit comments

Comments
 (0)