diff options
Diffstat (limited to 'py/lexer.c')
| -rw-r--r-- | py/lexer.c | 37 |
1 files changed, 28 insertions, 9 deletions
diff --git a/py/lexer.c b/py/lexer.c index a65df54ba6..8732d64362 100644 --- a/py/lexer.c +++ b/py/lexer.c @@ -32,8 +32,8 @@ #include <stdio.h> #include <assert.h> -#include "misc.h" #include "mpconfig.h" +#include "misc.h" #include "qstr.h" #include "lexer.h" @@ -502,19 +502,32 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs case 'v': c = 0x0b; break; case 'f': c = 0x0c; break; case 'r': c = 0x0d; break; + case 'u': + case 'U': + if (is_bytes) { + // b'\u1234' == b'\\u1234' + vstr_add_char(&lex->vstr, '\\'); + break; + } + // Otherwise fall through. case 'x': { uint num = 0; - if (!get_hex(lex, 2, &num)) { + if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) { // TODO error message assert(0); } c = num; break; } - case 'N': break; // TODO \N{name} only in strings - case 'u': break; // TODO \uxxxx only in strings - case 'U': break; // TODO \Uxxxxxxxx only in strings + case 'N': + // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the + // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly + // 3MB of text; even gzip-compressed and with minimal structure, it'll take + // roughly half a meg of storage. This form of Unicode escape may be added + // later on, but it's definitely not a priority right now. -- CJA 20140607 + assert(!"Unicode name escapes not supported"); + break; default: if (c >= '0' && c <= '7') { // Octal sequence, 1-3 chars @@ -533,7 +546,13 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs } } if (c != MP_LEXER_CHAR_EOF) { - vstr_add_char(&lex->vstr, c); + if (c < 0x110000 && !is_bytes) { + vstr_add_char(&lex->vstr, c); + } else if (c < 0x100 && is_bytes) { + vstr_add_byte(&lex->vstr, c); + } else { + assert(!"TODO: Throw an error, invalid escape code probably"); + } } } else { vstr_add_char(&lex->vstr, CUR_CHAR(lex)); @@ -694,10 +713,10 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs // need to check for this special token in many places in the compiler. // TODO improve speed of these string comparisons //for (int i = 0; tok_kw[i] != NULL; i++) { - for (int i = 0; i < ARRAY_SIZE(tok_kw); i++) { + for (int i = 0; i < MP_ARRAY_SIZE(tok_kw); i++) { if (str_strn_equal(tok_kw[i], tok->str, tok->len)) { - if (i == ARRAY_SIZE(tok_kw) - 1) { - // tok_kw[ARRAY_SIZE(tok_kw) - 1] == "__debug__" + if (i == MP_ARRAY_SIZE(tok_kw) - 1) { + // tok_kw[MP_ARRAY_SIZE(tok_kw) - 1] == "__debug__" tok->kind = (mp_optimise_value == 0 ? MP_TOKEN_KW_TRUE : MP_TOKEN_KW_FALSE); } else { tok->kind = MP_TOKEN_KW_FALSE + i; |