summaryrefslogtreecommitdiffstatshomepage
path: root/py/lexer.c
diff options
context:
space:
mode:
Diffstat (limited to 'py/lexer.c')
-rw-r--r--py/lexer.c10
1 files changed, 9 insertions, 1 deletions
diff --git a/py/lexer.c b/py/lexer.c
index 13c3a3e9b5..8a8875ed51 100644
--- a/py/lexer.c
+++ b/py/lexer.c
@@ -492,11 +492,19 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
}
}
if (c != MP_LEXER_EOF) {
+ #if MICROPY_PY_BUILTINS_STR_UNICODE
if (c < 0x110000 && !is_bytes) {
vstr_add_char(&lex->vstr, c);
} else if (c < 0x100 && is_bytes) {
vstr_add_byte(&lex->vstr, c);
- } else {
+ }
+ #else
+ // without unicode everything is just added as an 8-bit byte
+ if (c < 0x100) {
+ vstr_add_byte(&lex->vstr, c);
+ }
+ #endif
+ else {
assert(!"TODO: Throw an error, invalid escape code probably");
}
}