py: Be more precise about unicode type and disabled unicode behaviour.

1 files changed, 9 insertions, 1 deletions

diff --git a/py/lexer.c b/py/lexer.c
index 13c3a3e9b5..8a8875ed51 100644
--- a/py/lexer.c
+++ b/py/lexer.c
@@ -492,11 +492,19 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, bool first_token) {
                         }
                     }
                     if (c != MP_LEXER_EOF) {
+                        #if MICROPY_PY_BUILTINS_STR_UNICODE
                         if (c < 0x110000 && !is_bytes) {
                             vstr_add_char(&lex->vstr, c);
                         } else if (c < 0x100 && is_bytes) {
                             vstr_add_byte(&lex->vstr, c);
-                        } else {
+                        }
+                        #else
+                        // without unicode everything is just added as an 8-bit byte
+                        if (c < 0x100) {
+                            vstr_add_byte(&lex->vstr, c);
+                        }
+                        #endif
+                        else {
                             assert(!"TODO: Throw an error, invalid escape code probably");
                         }
                     }