diff options
48 files changed, 2378 insertions, 1743 deletions
diff --git a/mpy-cross/main.c b/mpy-cross/main.c index 7218921b31..a2daf7b22f 100644 --- a/mpy-cross/main.c +++ b/mpy-cross/main.c @@ -72,7 +72,8 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha #endif mp_parse_tree_t parse_tree = mp_parse(lex, MP_PARSE_FILE_INPUT); - mp_raw_code_t *rc = mp_compile_to_raw_code(&parse_tree, source_name, false); + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + mp_compiled_module_t cm = mp_compile_to_raw_code(&parse_tree, source_name, false, ctx); vstr_t vstr; vstr_init(&vstr, 16); @@ -83,7 +84,7 @@ STATIC int compile_and_save(const char *file, const char *output_file, const cha } else { vstr_add_str(&vstr, output_file); } - mp_raw_code_save_file(rc, vstr_null_terminated_str(&vstr)); + mp_raw_code_save_file(&cm, vstr_null_terminated_str(&vstr)); vstr_clear(&vstr); nlr_pop(); diff --git a/ports/unix/coverage.c b/ports/unix/coverage.c index 6b00cdfef2..a855237200 100644 --- a/ports/unix/coverage.c +++ b/ports/unix/coverage.c @@ -2,6 +2,7 @@ #include <string.h> #include "py/obj.h" +#include "py/objfun.h" #include "py/objstr.h" #include "py/runtime.h" #include "py/gc.h" @@ -449,7 +450,10 @@ STATIC mp_obj_t extra_coverage(void) { mp_printf(&mp_plat_print, "# VM\n"); // call mp_execute_bytecode with invalide bytecode (should raise NotImplementedError) + mp_module_context_t context; mp_obj_fun_bc_t fun_bc; + fun_bc.context = &context; + fun_bc.child_table = NULL; fun_bc.bytecode = (const byte *)"\x01"; // just needed for n_state mp_code_state_t *code_state = m_new_obj_var(mp_code_state_t, mp_obj_t, 1); code_state->fun_bc = &fun_bc; diff --git a/py/asmbase.c b/py/asmbase.c index 344e03e7a7..4a3fd089cb 100644 --- a/py/asmbase.c +++ b/py/asmbase.c @@ -61,7 +61,8 @@ void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { // all functions must go through this one to emit bytes // if as->pass < MP_ASM_PASS_EMIT, then this function just counts the number // of bytes needed and returns NULL, and callers should not store any data -uint8_t *mp_asm_base_get_cur_to_write_bytes(mp_asm_base_t *as, size_t num_bytes_to_write) { +uint8_t *mp_asm_base_get_cur_to_write_bytes(void *as_in, size_t num_bytes_to_write) { + mp_asm_base_t *as = as_in; uint8_t *c = NULL; if (as->pass == MP_ASM_PASS_EMIT) { assert(as->code_offset + num_bytes_to_write <= as->code_size); diff --git a/py/asmbase.h b/py/asmbase.h index 24c3af8679..960be7685f 100644 --- a/py/asmbase.h +++ b/py/asmbase.h @@ -45,7 +45,7 @@ typedef struct _mp_asm_base_t { void mp_asm_base_init(mp_asm_base_t *as, size_t max_num_labels); void mp_asm_base_deinit(mp_asm_base_t *as, bool free_code); void mp_asm_base_start_pass(mp_asm_base_t *as, int pass); -uint8_t *mp_asm_base_get_cur_to_write_bytes(mp_asm_base_t *as, size_t num_bytes_to_write); +uint8_t *mp_asm_base_get_cur_to_write_bytes(void *as, size_t num_bytes_to_write); void mp_asm_base_label_assign(mp_asm_base_t *as, size_t label); void mp_asm_base_align(mp_asm_base_t *as, unsigned int align); void mp_asm_base_data(mp_asm_base_t *as, unsigned int bytesize, uintptr_t val); @@ -29,9 +29,9 @@ #include <string.h> #include <assert.h> -#include "py/runtime.h" #include "py/bc0.h" #include "py/bc.h" +#include "py/objfun.h" #if MICROPY_DEBUG_VERBOSE // print debugging info #define DEBUG_PRINT (1) @@ -40,7 +40,23 @@ #define DEBUG_printf(...) (void)0 #endif -#if !MICROPY_PERSISTENT_CODE +void mp_encode_uint(void *env, mp_encode_uint_allocator_t allocator, mp_uint_t val) { + // We store each 7 bits in a separate byte, and that's how many bytes needed + byte buf[MP_ENCODE_UINT_MAX_BYTES]; + byte *p = buf + sizeof(buf); + // We encode in little-ending order, but store in big-endian, to help decoding + do { + *--p = val & 0x7f; + val >>= 7; + } while (val != 0); + byte *c = allocator(env, buf + sizeof(buf) - p); + if (c != NULL) { + while (p != buf + sizeof(buf) - 1) { + *c++ = *p++ | 0x80; + } + *c = *p; + } +} mp_uint_t mp_decode_uint(const byte **ptr) { mp_uint_t unum = 0; @@ -72,8 +88,6 @@ const byte *mp_decode_uint_skip(const byte *ptr) { return ptr; } -#endif - STATIC NORETURN void fun_pos_args_mismatch(mp_obj_fun_bc_t *f, size_t expected, size_t given) { #if MICROPY_ERROR_REPORTING <= MICROPY_ERROR_REPORTING_TERSE // generic message, used also for other argument issues @@ -107,8 +121,8 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // On entry code_state should be allocated somewhere (stack/heap) and // contain the following valid entries: // - code_state->fun_bc should contain a pointer to the function object -// - code_state->ip should contain the offset in bytes from the pointer -// code_state->fun_bc->bytecode to the entry n_state (0 for bytecode, non-zero for native) +// - code_state->ip should contain a pointer to the beginning of the prelude +// - code_state->n_state should be the number of objects in the local state void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args) { // This function is pretty complicated. It's main aim is to be efficient in speed and RAM // usage for the common case of positional only args. @@ -116,9 +130,6 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // get the function object that we want to set up (could be bytecode or native code) mp_obj_fun_bc_t *self = code_state->fun_bc; - // ip comes in as an offset into bytecode, so turn it into a true pointer - code_state->ip = self->bytecode + (size_t)code_state->ip; - #if MICROPY_STACKLESS code_state->prev = NULL; #endif @@ -134,6 +145,7 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // Decode prelude size_t n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args; MP_BC_PRELUDE_SIG_DECODE_INTO(code_state->ip, n_state_unused, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args); + MP_BC_PRELUDE_SIZE_DECODE(code_state->ip); (void)n_state_unused; (void)n_exc_stack_unused; @@ -194,14 +206,20 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw *var_pos_kw_args = dict; } - // get pointer to arg_names array - const mp_obj_t *arg_names = (const mp_obj_t *)self->const_table; - for (size_t i = 0; i < n_kw; i++) { // the keys in kwargs are expected to be qstr objects mp_obj_t wanted_arg_name = kwargs[2 * i]; + + // get pointer to arg_names array + const uint8_t *arg_names = code_state->ip; + arg_names = mp_decode_uint_skip(arg_names); + for (size_t j = 0; j < n_pos_args + n_kwonly_args; j++) { - if (wanted_arg_name == arg_names[j]) { + qstr arg_qstr = mp_decode_uint(&arg_names); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + arg_qstr = self->context->constants.qstr_table[arg_qstr]; + #endif + if (wanted_arg_name == MP_OBJ_NEW_QSTR(arg_qstr)) { if (code_state->state[n_state - 1 - j] != MP_OBJ_NULL) { mp_raise_msg_varg(&mp_type_TypeError, MP_ERROR_TEXT("function got multiple values for argument '%q'"), MP_OBJ_QSTR_VALUE(wanted_arg_name)); @@ -248,17 +266,25 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw // Check that all mandatory keyword args are specified // Fill in default kw args if we have them + const uint8_t *arg_names = mp_decode_uint_skip(code_state->ip); + for (size_t i = 0; i < n_pos_args; i++) { + arg_names = mp_decode_uint_skip(arg_names); + } for (size_t i = 0; i < n_kwonly_args; i++) { + qstr arg_qstr = mp_decode_uint(&arg_names); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + arg_qstr = self->context->constants.qstr_table[arg_qstr]; + #endif if (code_state->state[n_state - 1 - n_pos_args - i] == MP_OBJ_NULL) { mp_map_elem_t *elem = NULL; if ((scope_flags & MP_SCOPE_FLAG_DEFKWARGS) != 0) { - elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, arg_names[n_pos_args + i], MP_MAP_LOOKUP); + elem = mp_map_lookup(&((mp_obj_dict_t *)MP_OBJ_TO_PTR(self->extra_args[n_def_pos_args]))->map, MP_OBJ_NEW_QSTR(arg_qstr), MP_MAP_LOOKUP); } if (elem != NULL) { code_state->state[n_state - 1 - n_pos_args - i] = elem->value; } else { mp_raise_msg_varg(&mp_type_TypeError, - MP_ERROR_TEXT("function missing required keyword argument '%q'"), MP_OBJ_QSTR_VALUE(arg_names[n_pos_args + i])); + MP_ERROR_TEXT("function missing required keyword argument '%q'"), arg_qstr); } } } @@ -273,12 +299,8 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw } } - // read the size part of the prelude - const byte *ip = code_state->ip; - MP_BC_PRELUDE_SIZE_DECODE(ip); - - // jump over code info (source file and line-number mapping) - ip += n_info; + // jump over code info (source file, argument names and line-number mapping) + const uint8_t *ip = code_state->ip + n_info; // bytecode prelude: initialise closed over variables for (; n_cell; --n_cell) { @@ -287,11 +309,6 @@ void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw mp_obj_new_cell(code_state->state[n_state - 1 - local_num]); } - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - ip = MP_ALIGN(ip, sizeof(mp_uint_t)); - #endif - // now that we skipped over the prelude, set the ip for the VM code_state->ip = ip; @@ -28,7 +28,6 @@ #define MICROPY_INCLUDED_PY_BC_H #include "py/runtime.h" -#include "py/objfun.h" // bytecode layout: // @@ -50,7 +49,9 @@ // // source info section: // simple_name : var qstr -// source_file : var qstr +// argname0 : var qstr +// ... : var qstr +// argnameN : var qstr N = num_pos_args + num_kwonly_args - 1 // <line number info> // // closure section: @@ -58,19 +59,16 @@ // ... : byte // local_numN : byte N = n_cells-1 // -// <word alignment padding> only needed if bytecode contains pointers -// // <bytecode> // // // constant table layout: // -// argname0 : obj (qstr) -// ... : obj (qstr) -// argnameN : obj (qstr) N = num_pos_args + num_kwonly_args // const0 : obj // constN : obj +#define MP_ENCODE_UINT_MAX_BYTES ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) + #define MP_BC_PRELUDE_SIG_ENCODE(S, E, scope, out_byte, out_env) \ do { \ /*// Get values to store in prelude */ \ @@ -182,9 +180,9 @@ typedef struct _mp_bytecode_prelude_t { uint n_pos_args; uint n_kwonly_args; uint n_def_pos_args; - qstr qstr_block_name; - qstr qstr_source_file; + qstr qstr_block_name_idx; const byte *line_info; + const byte *line_info_top; const byte *opcodes; } mp_bytecode_prelude_t; @@ -198,12 +196,46 @@ typedef struct _mp_exc_stack_t { mp_obj_base_t *prev_exc; } mp_exc_stack_t; +// Constants associated with a module, to interface bytecode with runtime. +typedef struct _mp_module_constants_t { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + qstr_short_t *qstr_table; + #else + qstr source_file; + #endif + mp_obj_t *obj_table; +} mp_module_constants_t; + +// State associated with a module. +typedef struct _mp_module_context_t { + mp_obj_module_t module; + mp_module_constants_t constants; +} mp_module_context_t; + +// Outer level struct defining a compiled module. +typedef struct _mp_compiled_module_t { + const mp_module_context_t *context; + const struct _mp_raw_code_t *rc; + #if MICROPY_PERSISTENT_CODE_SAVE + bool has_native; + size_t n_qstr; + size_t n_obj; + #endif +} mp_compiled_module_t; + +// Outer level struct defining a frozen module. +typedef struct _mp_frozen_module_t { + const mp_module_constants_t constants; + const struct _mp_raw_code_t *rc; +} mp_frozen_module_t; + +// State for an executing function. typedef struct _mp_code_state_t { // The fun_bc entry points to the underlying function object that is being executed. // It is needed to access the start of bytecode and the const_table. // It is also needed to prevent the GC from reclaiming the bytecode during execution, // because the ip pointer below will always point to the interior of the bytecode. - mp_obj_fun_bc_t *fun_bc; + struct _mp_obj_fun_bc_t *fun_bc; const byte *ip; mp_obj_t *sp; uint16_t n_state; @@ -222,6 +254,10 @@ typedef struct _mp_code_state_t { // mp_exc_stack_t exc_state[0]; } mp_code_state_t; +// Allocator may return NULL, in which case data is not stored (can be used to compute size). +typedef uint8_t *(*mp_encode_uint_allocator_t)(void *env, size_t nbytes); + +void mp_encode_uint(void *env, mp_encode_uint_allocator_t allocator, mp_uint_t val); mp_uint_t mp_decode_uint(const byte **ptr); mp_uint_t mp_decode_uint_value(const byte *ptr); const byte *mp_decode_uint_skip(const byte *ptr); @@ -229,10 +265,10 @@ const byte *mp_decode_uint_skip(const byte *ptr); mp_vm_return_kind_t mp_execute_bytecode(mp_code_state_t *code_state, volatile mp_obj_t inject_exc); mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t func, size_t n_args, size_t n_kw, const mp_obj_t *args); void mp_setup_code_state(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_uint_t *const_table); -void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_uint_t *const_table); +void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *code, mp_uint_t len, const mp_module_constants_t *cm); +void mp_bytecode_print2(const mp_print_t *print, const byte *code, size_t len, const mp_module_constants_t *cm); const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip); -#define mp_bytecode_print_inst(print, code, const_table) mp_bytecode_print2(print, code, 1, const_table) +#define mp_bytecode_print_inst(print, code, x_table) mp_bytecode_print2(print, code, 1, x_table) // Helper macros to access pointer with least significant bits holding flags #define MP_TAGPTR_PTR(x) ((void *)((uintptr_t)(x) & ~((uintptr_t)3))) @@ -246,10 +282,26 @@ uint mp_opcode_format(const byte *ip, size_t *opcode_size, bool count_var_uint); #endif -static inline size_t mp_bytecode_get_source_line(const byte *line_info, size_t bc_offset) { +static inline void mp_module_context_alloc_tables(mp_module_context_t *context, size_t n_qstr, size_t n_obj) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + size_t nq = (n_qstr * sizeof(qstr_short_t) + sizeof(mp_uint_t) - 1) / sizeof(mp_uint_t); + size_t no = n_obj; + mp_uint_t *mem = m_new(mp_uint_t, nq + no); + context->constants.qstr_table = (void *)(mem); + context->constants.obj_table = (void *)(mem + nq); + #else + if (n_obj == 0) { + context->constants.obj_table = NULL; + } else { + context->constants.obj_table = m_new(mp_obj_t, n_obj); + } + #endif +} + +static inline size_t mp_bytecode_get_source_line(const byte *line_info, const byte *line_info_top, size_t bc_offset) { size_t source_line = 1; - size_t c; - while ((c = *line_info)) { + while (line_info < line_info_top) { + size_t c = *line_info; size_t b, l; if ((c & 0x80) == 0) { // 0b0LLBBBBB encoding diff --git a/py/builtinevex.c b/py/builtinevex.c index 800a20223a..4e1c6a66d1 100644 --- a/py/builtinevex.c +++ b/py/builtinevex.c @@ -54,7 +54,7 @@ STATIC mp_obj_t code_execute(mp_obj_code_t *self, mp_obj_dict_t *globals, mp_obj // the correct one if (mp_obj_is_type(self->module_fun, &mp_type_fun_bc)) { mp_obj_fun_bc_t *fun_bc = MP_OBJ_TO_PTR(self->module_fun); - fun_bc->globals = globals; + ((mp_module_context_t *)fun_bc->context)->module.globals = globals; } // execute code diff --git a/py/builtinimport.c b/py/builtinimport.c index 3e336633d9..094959f97d 100644 --- a/py/builtinimport.c +++ b/py/builtinimport.c @@ -146,28 +146,28 @@ STATIC mp_import_stat_t stat_top_level_dir_or_file(qstr mod_name, vstr_t *dest) } #if MICROPY_MODULE_FROZEN_STR || MICROPY_ENABLE_COMPILER -STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex) { +STATIC void do_load_from_lexer(mp_module_context_t *context, mp_lexer_t *lex) { #if MICROPY_PY___FILE__ qstr source_name = lex->source_name; - mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); + mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(source_name)); #endif // parse, compile and execute the module in its context - mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_obj_dict_t *mod_globals = context->module.globals; mp_parse_compile_execute(lex, MP_PARSE_FILE_INPUT, mod_globals, mod_globals); } #endif #if (MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD) || MICROPY_MODULE_FROZEN_MPY -STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, const char *source_name) { +STATIC void do_execute_raw_code(mp_module_context_t *context, const mp_raw_code_t *rc, const mp_module_context_t *mc, const char *source_name) { (void)source_name; #if MICROPY_PY___FILE__ - mp_store_attr(module_obj, MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name))); + mp_store_attr(MP_OBJ_FROM_PTR(&context->module), MP_QSTR___file__, MP_OBJ_NEW_QSTR(qstr_from_str(source_name))); #endif // execute the module in its context - mp_obj_dict_t *mod_globals = mp_obj_module_get_globals(module_obj); + mp_obj_dict_t *mod_globals = context->module.globals; // save context mp_obj_dict_t *volatile old_globals = mp_globals_get(); @@ -179,7 +179,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_obj_t module_fun = mp_make_function_from_raw_code(raw_code, MP_OBJ_NULL, MP_OBJ_NULL); + mp_obj_t module_fun = mp_make_function_from_raw_code(rc, mc, NULL); mp_call_function_0(module_fun); // finish nlr block, restore context @@ -195,7 +195,7 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code, co } #endif -STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { +STATIC void do_load(mp_module_context_t *module_obj, vstr_t *file) { #if MICROPY_MODULE_FROZEN || MICROPY_ENABLE_COMPILER || (MICROPY_PERSISTENT_CODE_LOAD && MICROPY_HAS_FILE_READER) const char *file_str = vstr_null_terminated_str(file); #endif @@ -222,7 +222,9 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // its data) in the list of frozen files, execute it. #if MICROPY_MODULE_FROZEN_MPY if (frozen_type == MP_FROZEN_MPY) { - do_execute_raw_code(module_obj, modref, file_str + frozen_path_prefix_len); + const mp_frozen_module_t *frozen = modref; + module_obj->constants = frozen->constants; + do_execute_raw_code(module_obj, frozen->rc, module_obj, file_str + frozen_path_prefix_len); return; } #endif @@ -234,8 +236,8 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) { // the correct format and, if so, load and execute the file. #if MICROPY_HAS_FILE_READER && MICROPY_PERSISTENT_CODE_LOAD if (file_str[file->len - 3] == 'm') { - mp_raw_code_t *raw_code = mp_raw_code_load_file(file_str); - do_execute_raw_code(module_obj, raw_code, file_str); + mp_compiled_module_t cm = mp_raw_code_load_file(file_str, module_obj); + do_execute_raw_code(module_obj, cm.rc, cm.context, file_str); return; } #endif @@ -434,7 +436,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, size_t orig_path_len = path->len; vstr_add_str(path, PATH_SEP_CHAR "__init__.py"); if (stat_file_py_or_mpy(path) == MP_IMPORT_STAT_FILE) { - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); } else { // No-op. Nothing to load. // mp_warning("%s is imported as namespace package", vstr_str(&path)); @@ -443,7 +445,7 @@ STATIC mp_obj_t process_import_at_level(qstr full_mod_name, qstr level_mod_name, path->len = orig_path_len; } else { // MP_IMPORT_STAT_FILE // File -- execute "path.(m)py". - do_load(module_obj, path); + do_load(MP_OBJ_TO_PTR(module_obj), path); // Note: This should be the last component in the import path. If // there are remaining components then it's an ImportError // because the current path(the module that was just loaded) is diff --git a/py/compile.c b/py/compile.c index 8ebcc2289f..92736e22e5 100644 --- a/py/compile.c +++ b/py/compile.c @@ -35,6 +35,7 @@ #include "py/compile.h" #include "py/runtime.h" #include "py/asmbase.h" +#include "py/nativeglue.h" #include "py/persistentcode.h" #if MICROPY_ENABLE_COMPILER @@ -88,7 +89,7 @@ typedef enum { #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER #define NATIVE_EMITTER(f) emit_native_table[mp_dynamic_compiler.native_arch]->emit_##f -#define NATIVE_EMITTER_TABLE emit_native_table[mp_dynamic_compiler.native_arch] +#define NATIVE_EMITTER_TABLE (emit_native_table[mp_dynamic_compiler.native_arch]) STATIC const emit_method_table_t *emit_native_table[] = { NULL, @@ -121,7 +122,7 @@ STATIC const emit_method_table_t *emit_native_table[] = { #else #error "unknown native emitter" #endif -#define NATIVE_EMITTER_TABLE &NATIVE_EMITTER(method_table) +#define NATIVE_EMITTER_TABLE (&NATIVE_EMITTER(method_table)) #endif #if MICROPY_EMIT_INLINE_ASM && MICROPY_DYNAMIC_COMPILER @@ -162,8 +163,6 @@ STATIC const emit_inline_asm_method_table_t *emit_asm_table[] = { // elements in this struct are ordered to make it compact typedef struct _compiler_t { - qstr source_file; - uint8_t is_repl; uint8_t pass; // holds enum type pass_kind_t uint8_t have_star; @@ -185,7 +184,10 @@ typedef struct _compiler_t { scope_t *scope_head; scope_t *scope_cur; + mp_emit_common_t emit_common; + emit_t *emit; // current emitter + emit_t *emit_bc; #if NEED_METHOD_TABLE const emit_method_table_t *emit_method_table; // current emit method table #endif @@ -196,6 +198,72 @@ typedef struct _compiler_t { #endif } compiler_t; +/******************************************************************************/ +// mp_emit_common_t helper functions +// These are defined here so they can be inlined, to reduce code size. + +STATIC void mp_emit_common_init(mp_emit_common_t *emit, qstr source_file) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + mp_map_init(&emit->qstr_map, 1); + + // add the source file as the first entry in the qstr table + mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(source_file), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + elem->value = MP_OBJ_NEW_SMALL_INT(0); + #endif +} + +STATIC void mp_emit_common_start_pass(mp_emit_common_t *emit, pass_kind_t pass) { + emit->pass = pass; + if (pass == MP_PASS_STACK_SIZE) { + emit->ct_cur_obj_base = emit->ct_cur_obj; + } else if (pass > MP_PASS_STACK_SIZE) { + emit->ct_cur_obj = emit->ct_cur_obj_base; + } + if (pass == MP_PASS_EMIT) { + if (emit->ct_cur_child == 0) { + emit->children = NULL; + } else { + emit->children = m_new0(mp_raw_code_t *, emit->ct_cur_child); + } + } + emit->ct_cur_child = 0; +} + +STATIC void mp_emit_common_finalise(mp_emit_common_t *emit, bool has_native_code) { + emit->ct_cur_obj += has_native_code; // allocate an additional slot for &mp_fun_table + emit->const_table = m_new0(mp_uint_t, emit->ct_cur_obj); + emit->ct_cur_obj = has_native_code; // reserve slot 0 for &mp_fun_table + #if MICROPY_EMIT_NATIVE + if (has_native_code) { + // store mp_fun_table pointer at the start of the constant table + emit->const_table[0] = (mp_uint_t)(uintptr_t)&mp_fun_table; + } + #endif +} + +STATIC void mp_emit_common_populate_module_context(mp_emit_common_t *emit, qstr source_file, mp_module_context_t *context) { + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + size_t qstr_map_used = emit->qstr_map.used; + mp_module_context_alloc_tables(context, qstr_map_used, emit->ct_cur_obj); + for (size_t i = 0; i < emit->qstr_map.alloc; ++i) { + if (mp_map_slot_is_filled(&emit->qstr_map, i)) { + size_t idx = MP_OBJ_SMALL_INT_VALUE(emit->qstr_map.table[i].value); + qstr qst = MP_OBJ_QSTR_VALUE(emit->qstr_map.table[i].key); + context->constants.qstr_table[idx] = qst; + } + } + #else + mp_module_context_alloc_tables(context, 0, emit->ct_cur_obj); + context->constants.source_file = source_file; + #endif + + if (emit->ct_cur_obj > 0) { + memcpy(context->constants.obj_table, emit->const_table, emit->ct_cur_obj * sizeof(mp_uint_t)); + } +} + +/******************************************************************************/ + STATIC void compile_error_set_line(compiler_t *comp, mp_parse_node_t pn) { // if the line of the error is unknown then try to update it from the pn if (comp->compile_error_line == 0 && MP_PARSE_NODE_IS_STRUCT(pn)) { @@ -246,7 +314,7 @@ STATIC void compile_decrease_except_level(compiler_t *comp) { } STATIC scope_t *scope_new_and_link(compiler_t *comp, scope_kind_t kind, mp_parse_node_t pn, uint emit_options) { - scope_t *scope = scope_new(kind, pn, comp->source_file, emit_options); + scope_t *scope = scope_new(kind, pn, emit_options); scope->parent = comp->scope_cur; scope->next = NULL; if (comp->scope_head == NULL) { @@ -290,7 +358,8 @@ STATIC void compile_generic_all_nodes(compiler_t *comp, mp_parse_node_struct_t * STATIC void compile_load_id(compiler_t *comp, qstr qst) { if (comp->pass == MP_PASS_SCOPE) { mp_emit_common_get_id_for_load(comp->scope_cur, qst); - } else { + } + { #if NEED_METHOD_TABLE mp_emit_common_id_op(comp->emit, &comp->emit_method_table->load_id, comp->scope_cur, qst); #else @@ -302,7 +371,8 @@ STATIC void compile_load_id(compiler_t *comp, qstr qst) { STATIC void compile_store_id(compiler_t *comp, qstr qst) { if (comp->pass == MP_PASS_SCOPE) { mp_emit_common_get_id_for_modification(comp->scope_cur, qst); - } else { + } + { #if NEED_METHOD_TABLE mp_emit_common_id_op(comp->emit, &comp->emit_method_table->store_id, comp->scope_cur, qst); #else @@ -314,7 +384,8 @@ STATIC void compile_store_id(compiler_t *comp, qstr qst) { STATIC void compile_delete_id(compiler_t *comp, qstr qst) { if (comp->pass == MP_PASS_SCOPE) { mp_emit_common_get_id_for_modification(comp->scope_cur, qst); - } else { + } + { #if NEED_METHOD_TABLE mp_emit_common_id_op(comp->emit, &comp->emit_method_table->delete_id, comp->scope_cur, qst); #else @@ -819,7 +890,7 @@ STATIC bool compile_built_in_decorator(compiler_t *comp, size_t name_len, mp_par compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid micropython decorator")); } - #if MICROPY_DYNAMIC_COMPILER + #if MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER if (*emit_options == MP_EMIT_OPT_NATIVE_PYTHON || *emit_options == MP_EMIT_OPT_VIPER) { if (emit_native_table[mp_dynamic_compiler.native_arch] == NULL) { compile_syntax_error(comp, name_nodes[1], MP_ERROR_TEXT("invalid arch")); @@ -2082,6 +2153,7 @@ STATIC void compile_lambdef(compiler_t *comp, mp_parse_node_struct_t *pns) { STATIC void compile_namedexpr_helper(compiler_t *comp, mp_parse_node_t pn_name, mp_parse_node_t pn_expr) { if (!MP_PARSE_NODE_IS_ID(pn_name)) { compile_syntax_error(comp, (mp_parse_node_t)pn_name, MP_ERROR_TEXT("can't assign to expression")); + return; // because pn_name is not a valid qstr (in compile_store_id below) } compile_node(comp, pn_expr); EMIT(dup_top); @@ -2814,6 +2886,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn // comes before a star, so counts as a positional parameter comp->scope_cur->num_pos_args += 1; } + mp_emit_common_use_qstr(&comp->emit_common, param_name); } else { assert(MP_PARSE_NODE_IS_STRUCT(pn)); pns = (mp_parse_node_struct_t *)pn; @@ -2827,6 +2900,7 @@ STATIC void compile_scope_func_lambda_param(compiler_t *comp, mp_parse_node_t pn // comes before a star, so counts as a positional parameter comp->scope_cur->num_pos_args += 1; } + mp_emit_common_use_qstr(&comp->emit_common, param_name); } else if (MP_PARSE_NODE_STRUCT_KIND(pns) == pn_star) { if (comp->have_star) { // more than one star @@ -2976,6 +3050,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { comp->pass = pass; comp->scope_cur = scope; comp->next_label = 0; + mp_emit_common_start_pass(&comp->emit_common, pass); EMIT_ARG(start_pass, pass, scope); reserve_labels_for_native(comp, 6); // used by native's start_pass @@ -2984,6 +3059,14 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { // they will be computed in this first pass scope->stack_size = 0; scope->exc_stack_size = 0; + + #if MICROPY_EMIT_NATIVE + if (scope->emit_options == MP_EMIT_OPT_NATIVE_PYTHON || scope->emit_options == MP_EMIT_OPT_VIPER) { + // allow native code to perfom basic tasks during the pass scope + // note: the first argument passed here is mp_emit_common_t, not the native emitter context + NATIVE_EMITTER_TABLE->start_pass((void *)&comp->emit_common, comp->pass, scope); + } + #endif } // compile @@ -3064,6 +3147,7 @@ STATIC void compile_scope(compiler_t *comp, scope_t *scope, pass_kind_t pass) { if (comp->pass == MP_PASS_SCOPE) { scope_find_or_add_id(comp->scope_cur, qstr_arg, ID_INFO_KIND_LOCAL); scope->num_pos_args = 1; + mp_emit_common_use_qstr(&comp->emit_common, MP_QSTR__star_); } // Set the source line number for the start of the comprehension @@ -3296,9 +3380,10 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind f, mp_asm_base_get_code_size((mp_asm_base_t *)comp->emit_inline_asm), NULL, #if MICROPY_PERSISTENT_CODE_SAVE - 0, 0, 0, 0, NULL, + 0, + 0, 0, NULL, #endif - comp->scope_cur->num_pos_args, 0, type_sig); + 0, comp->scope_cur->num_pos_args, type_sig); } } @@ -3310,7 +3395,7 @@ STATIC void compile_scope_inline_asm(compiler_t *comp, scope_t *scope, pass_kind } #endif -STATIC void scope_compute_things(scope_t *scope) { +STATIC void scope_compute_things(scope_t *scope, mp_emit_common_t *emit_common) { // in MicroPython we put the *x parameter after all other parameters (except **y) if (scope->scope_flags & MP_SCOPE_FLAG_VARARGS) { id_info_t *id_param = NULL; @@ -3399,6 +3484,7 @@ STATIC void scope_compute_things(scope_t *scope) { } scope->num_pos_args += num_free; // free vars are counted as params for passing them into the function scope->num_locals += num_free; + mp_emit_common_use_qstr(emit_common, MP_QSTR__star_); } } } @@ -3406,15 +3492,15 @@ STATIC void scope_compute_things(scope_t *scope) { #if !MICROPY_PERSISTENT_CODE_SAVE STATIC #endif -mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *context) { // put compiler state on the stack, it's relatively small compiler_t comp_state = {0}; compiler_t *comp = &comp_state; - comp->source_file = source_file; comp->is_repl = is_repl; comp->break_label = INVALID_LABEL; comp->continue_label = INVALID_LABEL; + mp_emit_common_init(&comp->emit_common, source_file); // create the module scope #if MICROPY_EMIT_NATIVE @@ -3425,10 +3511,11 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f scope_t *module_scope = scope_new_and_link(comp, SCOPE_MODULE, parse_tree->root, emit_opt); // create standard emitter; it's used at least for MP_PASS_SCOPE - emit_t *emit_bc = emit_bc_new(); + emit_t *emit_bc = emit_bc_new(&comp->emit_common); - // compile pass 1 + // compile MP_PASS_SCOPE comp->emit = emit_bc; + comp->emit_bc = emit_bc; #if MICROPY_EMIT_NATIVE comp->emit_method_table = &emit_bc_method_table; #endif @@ -3458,14 +3545,24 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f } // compute some things related to scope and identifiers + bool has_native_code = false; for (scope_t *s = comp->scope_head; s != NULL && comp->compile_error == MP_OBJ_NULL; s = s->next) { - scope_compute_things(s); + #if MICROPY_EMIT_NATIVE + if (s->emit_options == MP_EMIT_OPT_NATIVE_PYTHON || s->emit_options == MP_EMIT_OPT_VIPER) { + has_native_code = true; + } + #endif + + scope_compute_things(s, &comp->emit_common); } // set max number of labels now that it's calculated emit_bc_set_max_num_labels(emit_bc, max_num_labels); - // compile pass 2 and 3 + // finalise and allocate the constant table + mp_emit_common_finalise(&comp->emit_common, has_native_code); + + // compile MP_PASS_STACK_SIZE, MP_PASS_CODE_SIZE, MP_PASS_EMIT #if MICROPY_EMIT_NATIVE emit_t *emit_native = NULL; #endif @@ -3505,7 +3602,7 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f case MP_EMIT_OPT_NATIVE_PYTHON: case MP_EMIT_OPT_VIPER: if (emit_native == NULL) { - emit_native = NATIVE_EMITTER(new)(&comp->compile_error, &comp->next_label, max_num_labels); + emit_native = NATIVE_EMITTER(new)(&comp->emit_common, &comp->compile_error, &comp->next_label, max_num_labels); } comp->emit_method_table = NATIVE_EMITTER_TABLE; comp->emit = emit_native; @@ -3540,10 +3637,33 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f // number for the start of this scope compile_error_set_line(comp, comp->scope_cur->pn); // add a traceback to the exception using relevant source info - mp_obj_exception_add_traceback(comp->compile_error, comp->source_file, + mp_obj_exception_add_traceback(comp->compile_error, source_file, comp->compile_error_line, comp->scope_cur->simple_name); } + // construct the global qstr/const table for this module + mp_compiled_module_t cm; + cm.rc = module_scope->raw_code; + cm.context = context; + #if MICROPY_PERSISTENT_CODE_SAVE + cm.has_native = has_native_code; + cm.n_qstr = comp->emit_common.qstr_map.used; + cm.n_obj = comp->emit_common.ct_cur_obj; + #endif + if (comp->compile_error == MP_OBJ_NULL) { + mp_emit_common_populate_module_context(&comp->emit_common, source_file, context); + + #if MICROPY_DEBUG_PRINTERS + // now that the module context is valid, the raw codes can be printed + if (mp_verbose_flag >= 2) { + for (scope_t *s = comp->scope_head; s != NULL; s = s->next) { + mp_raw_code_t *rc = s->raw_code; + mp_bytecode_print(&mp_plat_print, rc, rc->fun_data, rc->fun_data_len, &cm.context->constants); + } + } + #endif + } + // free the emitters emit_bc_free(emit_bc); @@ -3562,7 +3682,6 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f mp_parse_tree_clear(parse_tree); // free the scopes - mp_raw_code_t *outer_raw_code = module_scope->raw_code; for (scope_t *s = module_scope; s;) { scope_t *next = s->next; scope_free(s); @@ -3571,15 +3690,17 @@ mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_f if (comp->compile_error != MP_OBJ_NULL) { nlr_raise(comp->compile_error); - } else { - return outer_raw_code; } + + return cm; } mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl) { - mp_raw_code_t *rc = mp_compile_to_raw_code(parse_tree, source_file, is_repl); + mp_module_context_t *context = m_new_obj(mp_module_context_t); + context->module.globals = mp_globals_get(); + mp_compiled_module_t cm = mp_compile_to_raw_code(parse_tree, source_file, is_repl, context); // return function that executes the outer module - return mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); + return mp_make_function_from_raw_code(cm.rc, cm.context, NULL); } #endif // MICROPY_ENABLE_COMPILER diff --git a/py/compile.h b/py/compile.h index 1ad1f5e9cd..ae87bf2a04 100644 --- a/py/compile.h +++ b/py/compile.h @@ -32,11 +32,12 @@ // the compiler will raise an exception if an error occurred // the compiler will clear the parse tree before it returns +// mp_globals_get() will be used for the context mp_obj_t mp_compile(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); #if MICROPY_PERSISTENT_CODE_SAVE // this has the same semantics as mp_compile -mp_raw_code_t *mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl); +mp_compiled_module_t mp_compile_to_raw_code(mp_parse_tree_t *parse_tree, qstr source_file, bool is_repl, mp_module_context_t *globals); #endif // this is implemented in runtime.c diff --git a/py/dynruntime.h b/py/dynruntime.h index fdb91ed37e..b9dca60d13 100644 --- a/py/dynruntime.h +++ b/py/dynruntime.h @@ -30,6 +30,7 @@ // MicroPython runtime API defined in py/obj.h and py/runtime.h. #include "py/nativeglue.h" +#include "py/objfun.h" #include "py/objstr.h" #include "py/objtype.h" @@ -177,8 +178,8 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { #define mp_unary_op(op, obj) (mp_fun_table.unary_op((op), (obj))) #define mp_binary_op(op, lhs, rhs) (mp_fun_table.binary_op((op), (lhs), (rhs))) -#define mp_make_function_from_raw_code(rc, def_args, def_kw_args) \ - (mp_fun_table.make_function_from_raw_code((rc), (def_args), (def_kw_args))) +#define mp_make_function_from_raw_code(rc, context, def_args) \ + (mp_fun_table.make_function_from_raw_code((rc), (context), (def_args))) #define mp_call_function_n_kw(fun, n_args, n_kw, args) \ (mp_fun_table.call_function_n_kw((fun), (n_args) | ((n_kw) << 8), args)) @@ -187,11 +188,10 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { (mp_fun_table.arg_check_num_sig((n_args), (n_kw), MP_OBJ_FUN_MAKE_SIG((n_args_min), (n_args_max), (takes_kw)))) #define MP_DYNRUNTIME_INIT_ENTRY \ - mp_obj_t old_globals = mp_fun_table.swap_globals(self->globals); \ + mp_obj_t old_globals = mp_fun_table.swap_globals(self->context->module.globals); \ mp_raw_code_t rc; \ rc.kind = MP_CODE_NATIVE_VIPER; \ rc.scope_flags = 0; \ - rc.const_table = (void *)self->const_table; \ (void)rc; #define MP_DYNRUNTIME_INIT_EXIT \ @@ -199,7 +199,7 @@ static inline mp_obj_t mp_obj_len_dyn(mp_obj_t o) { return mp_const_none; #define MP_DYNRUNTIME_MAKE_FUNCTION(f) \ - (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), MP_OBJ_NULL, MP_OBJ_NULL)) + (mp_make_function_from_raw_code((rc.fun_data = (f), &rc), self->context, NULL)) #define mp_import_name(name, fromlist, level) \ (mp_fun_table.import_name((name), (fromlist), (level))) @@ -92,6 +92,18 @@ typedef enum { typedef struct _emit_t emit_t; +typedef struct _mp_emit_common_t { + pass_kind_t pass; + uint16_t ct_cur_obj_base; + uint16_t ct_cur_obj; + uint16_t ct_cur_child; + mp_uint_t *const_table; + mp_raw_code_t **children; + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + mp_map_t qstr_map; + #endif +} mp_emit_common_t; + typedef struct _mp_emit_method_table_id_ops_t { void (*local)(emit_t *emit, qstr qst, mp_uint_t local_num, int kind); void (*global)(emit_t *emit, qstr qst, int kind); @@ -99,7 +111,7 @@ typedef struct _mp_emit_method_table_id_ops_t { typedef struct _emit_method_table_t { #if MICROPY_DYNAMIC_COMPILER - emit_t *(*emit_new)(mp_obj_t * error_slot, uint *label_slot, mp_uint_t max_num_labels); + emit_t *(*emit_new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); void (*emit_free)(emit_t *emit); #endif @@ -161,6 +173,28 @@ typedef struct _emit_method_table_t { void (*end_except_handler)(emit_t *emit); } emit_method_table_t; +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst); +#else +static inline qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) { + return qst; +} +#endif + +static inline size_t mp_emit_common_alloc_const_obj(mp_emit_common_t *emit, mp_obj_t obj) { + if (emit->pass == MP_PASS_EMIT) { + emit->const_table[emit->ct_cur_obj] = (mp_uint_t)obj; + } + return emit->ct_cur_obj++; +} + +static inline size_t mp_emit_common_alloc_const_child(mp_emit_common_t *emit, mp_raw_code_t *rc) { + if (emit->pass == MP_PASS_EMIT) { + emit->children[emit->ct_cur_child] = rc; + } + return emit->ct_cur_child++; +} + static inline void mp_emit_common_get_id_for_load(scope_t *scope, qstr qst) { scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT); } @@ -180,13 +214,13 @@ extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_load_id_ops; extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_store_id_ops; extern const mp_emit_method_table_id_ops_t mp_emit_bc_method_table_delete_id_ops; -emit_t *emit_bc_new(void); -emit_t *emit_native_x64_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_x86_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_thumb_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_arm_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_xtensa_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); -emit_t *emit_native_xtensawin_new(mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_bc_new(mp_emit_common_t *emit_common); +emit_t *emit_native_x64_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_x86_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_thumb_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_arm_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_xtensa_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); +emit_t *emit_native_xtensawin_new(mp_emit_common_t *emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels); void emit_bc_set_max_num_labels(emit_t *emit, mp_uint_t max_num_labels); diff --git a/py/emitbc.c b/py/emitbc.c index ca74046033..c04701ca78 100644 --- a/py/emitbc.c +++ b/py/emitbc.c @@ -36,8 +36,7 @@ #if MICROPY_ENABLE_COMPILER -#define BYTES_FOR_INT ((MP_BYTES_PER_OBJ_WORD * 8 + 6) / 7) -#define DUMMY_DATA_SIZE (BYTES_FOR_INT) +#define DUMMY_DATA_SIZE (MP_ENCODE_UINT_MAX_BYTES) struct _emit_t { // Accessed as mp_obj_t, so must be aligned as such, and we rely on the @@ -50,6 +49,7 @@ struct _emit_t { int stack_size; + mp_emit_common_t *emit_common; scope_t *scope; mp_uint_t last_source_line_offset; @@ -66,17 +66,11 @@ struct _emit_t { size_t n_info; size_t n_cell; - - #if MICROPY_PERSISTENT_CODE - uint16_t ct_cur_obj; - uint16_t ct_num_obj; - uint16_t ct_cur_raw_code; - #endif - mp_uint_t *const_table; }; -emit_t *emit_bc_new(void) { +emit_t *emit_bc_new(mp_emit_common_t *emit_common) { emit_t *emit = m_new0(emit_t, 1); + emit->emit_common = emit_common; return emit; } @@ -90,26 +84,9 @@ void emit_bc_free(emit_t *emit) { m_del_obj(emit_t, emit); } -typedef byte *(*emit_allocator_t)(emit_t *emit, int nbytes); - -STATIC void emit_write_uint(emit_t *emit, emit_allocator_t allocator, mp_uint_t val) { - // We store each 7 bits in a separate byte, and that's how many bytes needed - byte buf[BYTES_FOR_INT]; - byte *p = buf + sizeof(buf); - // We encode in little-ending order, but store in big-endian, to help decoding - do { - *--p = val & 0x7f; - val >>= 7; - } while (val != 0); - byte *c = allocator(emit, buf + sizeof(buf) - p); - while (p != buf + sizeof(buf) - 1) { - *c++ = *p++ | 0x80; - } - *c = *p; -} - // all functions must go through this one to emit code info -STATIC byte *emit_get_cur_to_write_code_info(emit_t *emit, int num_bytes_to_write) { +STATIC uint8_t *emit_get_cur_to_write_code_info(void *emit_in, size_t num_bytes_to_write) { + emit_t *emit = emit_in; if (emit->pass < MP_PASS_EMIT) { emit->code_info_offset += num_bytes_to_write; return emit->dummy_data; @@ -126,14 +103,7 @@ STATIC void emit_write_code_info_byte(emit_t *emit, byte val) { } STATIC void emit_write_code_info_qstr(emit_t *emit, qstr qst) { - #if MICROPY_PERSISTENT_CODE - assert((qst >> 16) == 0); - byte *c = emit_get_cur_to_write_code_info(emit, 2); - c[0] = qst; - c[1] = qst >> 8; - #else - emit_write_uint(emit, emit_get_cur_to_write_code_info, qst); - #endif + mp_encode_uint(emit, emit_get_cur_to_write_code_info, mp_emit_common_use_qstr(emit->emit_common, qst)); } #if MICROPY_ENABLE_SOURCE_LINE @@ -166,7 +136,8 @@ STATIC void emit_write_code_info_bytes_lines(emit_t *emit, mp_uint_t bytes_to_sk #endif // all functions must go through this one to emit byte code -STATIC byte *emit_get_cur_to_write_bytecode(emit_t *emit, int num_bytes_to_write) { +STATIC uint8_t *emit_get_cur_to_write_bytecode(void *emit_in, size_t num_bytes_to_write) { + emit_t *emit = emit_in; if (emit->pass < MP_PASS_EMIT) { emit->bytecode_offset += num_bytes_to_write; return emit->dummy_data; @@ -189,12 +160,12 @@ STATIC void emit_write_bytecode_byte(emit_t *emit, int stack_adj, byte b1) { c[0] = b1; } -// Similar to emit_write_bytecode_uint(), just some extra handling to encode sign +// Similar to mp_encode_uint(), just some extra handling to encode sign STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, mp_int_t num) { emit_write_bytecode_byte(emit, stack_adj, b1); // We store each 7 bits in a separate byte, and that's how many bytes needed - byte buf[BYTES_FOR_INT]; + byte buf[MP_ENCODE_UINT_MAX_BYTES]; byte *p = buf + sizeof(buf); // We encode in little-ending order, but store in big-endian, to help decoding do { @@ -218,61 +189,25 @@ STATIC void emit_write_bytecode_byte_int(emit_t *emit, int stack_adj, byte b1, m STATIC void emit_write_bytecode_byte_uint(emit_t *emit, int stack_adj, byte b, mp_uint_t val) { emit_write_bytecode_byte(emit, stack_adj, b); - emit_write_uint(emit, emit_get_cur_to_write_bytecode, val); + mp_encode_uint(emit, emit_get_cur_to_write_bytecode, val); } -#if MICROPY_PERSISTENT_CODE -STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n, mp_uint_t c) { - if (emit->pass == MP_PASS_EMIT) { - emit->const_table[n] = c; - } +STATIC void emit_write_bytecode_byte_const(emit_t *emit, int stack_adj, byte b, mp_uint_t n) { emit_write_bytecode_byte_uint(emit, stack_adj, b, n); } -#endif STATIC void emit_write_bytecode_byte_qstr(emit_t *emit, int stack_adj, byte b, qstr qst) { - #if MICROPY_PERSISTENT_CODE - assert((qst >> 16) == 0); - mp_emit_bc_adjust_stack_size(emit, stack_adj); - byte *c = emit_get_cur_to_write_bytecode(emit, 3); - c[0] = b; - c[1] = qst; - c[2] = qst >> 8; - #else - emit_write_bytecode_byte_uint(emit, stack_adj, b, qst); - #endif + emit_write_bytecode_byte_uint(emit, stack_adj, b, mp_emit_common_use_qstr(emit->emit_common, qst)); } STATIC void emit_write_bytecode_byte_obj(emit_t *emit, int stack_adj, byte b, mp_obj_t obj) { - #if MICROPY_PERSISTENT_CODE emit_write_bytecode_byte_const(emit, stack_adj, b, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_cur_obj++, (mp_uint_t)obj); - #else - // aligns the pointer so it is friendly to GC - emit_write_bytecode_byte(emit, stack_adj, b); - emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(mp_obj_t)); - mp_obj_t *c = (mp_obj_t *)emit_get_cur_to_write_bytecode(emit, sizeof(mp_obj_t)); - // Verify thar c is already uint-aligned - assert(c == MP_ALIGN(c, sizeof(mp_obj_t))); - *c = obj; - #endif + mp_emit_common_alloc_const_obj(emit->emit_common, obj)); } -STATIC void emit_write_bytecode_byte_raw_code(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) { - #if MICROPY_PERSISTENT_CODE +STATIC void emit_write_bytecode_byte_child(emit_t *emit, int stack_adj, byte b, mp_raw_code_t *rc) { emit_write_bytecode_byte_const(emit, stack_adj, b, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_num_obj + emit->ct_cur_raw_code++, (mp_uint_t)(uintptr_t)rc); - #else - // aligns the pointer so it is friendly to GC - emit_write_bytecode_byte(emit, stack_adj, b); - emit->bytecode_offset = (size_t)MP_ALIGN(emit->bytecode_offset, sizeof(void *)); - void **c = (void **)emit_get_cur_to_write_bytecode(emit, sizeof(void *)); - // Verify thar c is already uint-aligned - assert(c == MP_ALIGN(c, sizeof(void *))); - *c = rc; - #endif + mp_emit_common_alloc_const_child(emit->emit_common, rc)); #if MICROPY_PY_SYS_SETTRACE rc->line_of_definition = emit->last_source_line; #endif @@ -343,27 +278,19 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { } // Write number of cells and size of the source code info - if (pass >= MP_PASS_CODE_SIZE) { - MP_BC_PRELUDE_SIZE_ENCODE(emit->n_info, emit->n_cell, emit_write_code_info_byte, emit); + if (emit->pass >= MP_PASS_CODE_SIZE) { + size_t n_info = emit->n_info; + size_t n_cell = emit->n_cell; + MP_BC_PRELUDE_SIZE_ENCODE(n_info, n_cell, emit_write_code_info_byte, emit); } emit->n_info = emit->code_info_offset; - // Write the name and source file of this function. + // Write the name of this function. emit_write_code_info_qstr(emit, scope->simple_name); - emit_write_code_info_qstr(emit, scope->source_file); - - #if MICROPY_PERSISTENT_CODE - emit->ct_cur_obj = 0; - emit->ct_cur_raw_code = 0; - #endif - - if (pass == MP_PASS_EMIT) { - // Write argument names (needed to resolve positional args passed as - // keywords). We store them as full word-sized objects for efficient access - // in mp_setup_code_state this is the start of the prelude and is guaranteed - // to be aligned on a word boundary. + // Write argument names, needed to resolve positional args passed as keywords. + { // For a given argument position (indexed by i) we need to find the // corresponding id_info which is a parameter, as it has the correct // qstr name to use as the argument name. Note that it's not a simple @@ -383,7 +310,7 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { break; } } - emit->const_table[i] = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); + emit_write_code_info_qstr(emit, qst); } } } @@ -396,8 +323,6 @@ void mp_emit_bc_end_pass(emit_t *emit) { // check stack is back to zero size assert(emit->stack_size == 0); - emit_write_code_info_byte(emit, 0); // end of line number info - // Calculate size of source code info section emit->n_info = emit->code_info_offset - emit->n_info; @@ -412,39 +337,20 @@ void mp_emit_bc_end_pass(emit_t *emit) { } } - #if MICROPY_PERSISTENT_CODE - assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->ct_num_obj == emit->ct_cur_obj)); - emit->ct_num_obj = emit->ct_cur_obj; - #endif - if (emit->pass == MP_PASS_CODE_SIZE) { - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - emit->code_info_offset = (size_t)MP_ALIGN(emit->code_info_offset, sizeof(mp_uint_t)); - #endif - // calculate size of total code-info + bytecode, in bytes emit->code_info_size = emit->code_info_offset; emit->bytecode_size = emit->bytecode_offset; emit->code_base = m_new0(byte, emit->code_info_size + emit->bytecode_size); - #if MICROPY_PERSISTENT_CODE - emit->const_table = m_new0(mp_uint_t, - emit->scope->num_pos_args + emit->scope->num_kwonly_args - + emit->ct_cur_obj + emit->ct_cur_raw_code); - #else - emit->const_table = m_new0(mp_uint_t, - emit->scope->num_pos_args + emit->scope->num_kwonly_args); - #endif - } else if (emit->pass == MP_PASS_EMIT) { mp_emit_glue_assign_bytecode(emit->scope->raw_code, emit->code_base, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS emit->code_info_size + emit->bytecode_size, #endif - emit->const_table, + emit->emit_common->children, #if MICROPY_PERSISTENT_CODE_SAVE - emit->ct_cur_obj, emit->ct_cur_raw_code, + emit->emit_common->ct_cur_child, #endif emit->scope->scope_flags); } @@ -783,21 +689,21 @@ void mp_emit_bc_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_right) { void mp_emit_bc_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { if (n_pos_defaults == 0 && n_kw_defaults == 0) { - emit_write_bytecode_byte_raw_code(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code); + emit_write_bytecode_byte_child(emit, 1, MP_BC_MAKE_FUNCTION, scope->raw_code); } else { - emit_write_bytecode_byte_raw_code(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code); + emit_write_bytecode_byte_child(emit, -1, MP_BC_MAKE_FUNCTION_DEFARGS, scope->raw_code); } } void mp_emit_bc_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { if (n_pos_defaults == 0 && n_kw_defaults == 0) { int stack_adj = -n_closed_over + 1; - emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code); + emit_write_bytecode_byte_child(emit, stack_adj, MP_BC_MAKE_CLOSURE, scope->raw_code); emit_write_bytecode_raw_byte(emit, n_closed_over); } else { assert(n_closed_over <= 255); int stack_adj = -2 - (mp_int_t)n_closed_over + 1; - emit_write_bytecode_byte_raw_code(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code); + emit_write_bytecode_byte_child(emit, stack_adj, MP_BC_MAKE_CLOSURE_DEFARGS, scope->raw_code); emit_write_bytecode_raw_byte(emit, n_closed_over); } } diff --git a/py/emitcommon.c b/py/emitcommon.c index 791bf398ab..c1780d2db9 100644 --- a/py/emitcommon.c +++ b/py/emitcommon.c @@ -30,6 +30,17 @@ #if MICROPY_ENABLE_COMPILER +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +qstr_short_t mp_emit_common_use_qstr(mp_emit_common_t *emit, qstr qst) { + mp_map_elem_t *elem = mp_map_lookup(&emit->qstr_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); + if (elem->value == MP_OBJ_NULL) { + assert(emit->pass == MP_PASS_SCOPE); + elem->value = MP_OBJ_NEW_SMALL_INT(emit->qstr_map.used - 1); + } + return MP_OBJ_SMALL_INT_VALUE(elem->value); +} +#endif + void mp_emit_common_get_id_for_modification(scope_t *scope, qstr qst) { // name adding/lookup id_info_t *id = scope_find_or_add_id(scope, qst, ID_INFO_KIND_GLOBAL_IMPLICIT); diff --git a/py/emitglue.c b/py/emitglue.c index 09b48682ff..cd902838af 100644 --- a/py/emitglue.c +++ b/py/emitglue.c @@ -34,6 +34,7 @@ #include "py/emitglue.h" #include "py/runtime0.h" #include "py/bc.h" +#include "py/objfun.h" #include "py/profile.h" #if MICROPY_DEBUG_VERBOSE // print debugging info @@ -63,20 +64,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS size_t len, #endif - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t n_obj, uint16_t n_raw_code, + size_t n_children, #endif mp_uint_t scope_flags) { rc->kind = MP_CODE_BYTECODE; rc->scope_flags = scope_flags; rc->fun_data = code; - rc->const_table = const_table; - #if MICROPY_PERSISTENT_CODE_SAVE + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS rc->fun_data_len = len; - rc->n_obj = n_obj; - rc->n_raw_code = n_raw_code; + #endif + rc->children = children; + + #if MICROPY_PERSISTENT_CODE_SAVE + rc->n_children = n_children; #endif #if MICROPY_PY_SYS_SETTRACE @@ -85,26 +88,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #endif #ifdef DEBUG_PRINT - #if !MICROPY_DEBUG_PRINTERS + #if !(MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS) const size_t len = 0; #endif DEBUG_printf("assign byte code: code=%p len=" UINT_FMT " flags=%x\n", code, len, (uint)scope_flags); #endif - #if MICROPY_DEBUG_PRINTERS - if (mp_verbose_flag >= 2) { - mp_bytecode_print(&mp_plat_print, rc, code, len, const_table); - } - #endif } #if MICROPY_EMIT_MACHINE_CODE -void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, const mp_uint_t *const_table, +void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE + size_t n_children, uint16_t prelude_offset, - uint16_t n_obj, uint16_t n_raw_code, uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link, #endif - mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig) { + mp_uint_t scope_flags, mp_uint_t n_pos_args, mp_uint_t type_sig) { assert(kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER || kind == MP_CODE_NATIVE_ASM); @@ -135,20 +134,24 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void rc->kind = kind; rc->scope_flags = scope_flags; - rc->n_pos_args = n_pos_args; rc->fun_data = fun_data; - rc->const_table = const_table; - rc->type_sig = type_sig; - #if MICROPY_PERSISTENT_CODE_SAVE + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS rc->fun_data_len = fun_len; + #endif + rc->children = children; + + #if MICROPY_PERSISTENT_CODE_SAVE + rc->n_children = n_children; rc->prelude_offset = prelude_offset; - rc->n_obj = n_obj; - rc->n_raw_code = n_raw_code; rc->n_qstr = n_qstr; rc->qstr_link = qstr_link; #endif + // These two entries are only needed for MP_CODE_NATIVE_ASM. + rc->n_pos_args = n_pos_args; + rc->type_sig = type_sig; + #ifdef DEBUG_PRINT DEBUG_printf("assign native: kind=%d fun=%p len=" UINT_FMT " n_pos_args=" UINT_FMT " flags=%x\n", kind, fun_data, fun_len, n_pos_args, (uint)scope_flags); for (mp_uint_t i = 0; i < fun_len; i++) { @@ -170,15 +173,15 @@ void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void } #endif -mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args) { +mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, const mp_obj_t *def_args) { DEBUG_OP_printf("make_function_from_raw_code %p\n", rc); assert(rc != NULL); // def_args must be MP_OBJ_NULL or a tuple - assert(def_args == MP_OBJ_NULL || mp_obj_is_type(def_args, &mp_type_tuple)); + assert(def_args == NULL || def_args[0] == MP_OBJ_NULL || mp_obj_is_type(def_args[0], &mp_type_tuple)); // def_kw_args must be MP_OBJ_NULL or a dict - assert(def_kw_args == MP_OBJ_NULL || mp_obj_is_type(def_kw_args, &mp_type_dict)); + assert(def_args == NULL || def_args[1] == MP_OBJ_NULL || mp_obj_is_type(def_args[1], &mp_type_dict)); // make the function, depending on the raw code kind mp_obj_t fun; @@ -186,7 +189,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar #if MICROPY_EMIT_NATIVE case MP_CODE_NATIVE_PY: case MP_CODE_NATIVE_VIPER: - fun = mp_obj_new_fun_native(def_args, def_kw_args, rc->fun_data, rc->const_table); + fun = mp_obj_new_fun_native(def_args, rc->fun_data, context, rc->children); // Check for a generator function, and if so change the type of the object if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_native_gen_wrap; @@ -201,7 +204,7 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar default: // rc->kind should always be set and BYTECODE is the only remaining case assert(rc->kind == MP_CODE_BYTECODE); - fun = mp_obj_new_fun_bc(def_args, def_kw_args, rc->fun_data, rc->const_table); + fun = mp_obj_new_fun_bc(def_args, rc->fun_data, context, rc->children); // check for generator functions and if so change the type of the object if ((rc->scope_flags & MP_SCOPE_FLAG_GENERATOR) != 0) { ((mp_obj_base_t *)MP_OBJ_TO_PTR(fun))->type = &mp_type_gen_wrap; @@ -218,16 +221,16 @@ mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_ar return fun; } -mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args) { +mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, mp_uint_t n_closed_over, const mp_obj_t *args) { DEBUG_OP_printf("make_closure_from_raw_code %p " UINT_FMT " %p\n", rc, n_closed_over, args); // make function object mp_obj_t ffun; if (n_closed_over & 0x100) { // default positional and keyword args given - ffun = mp_make_function_from_raw_code(rc, args[0], args[1]); + ffun = mp_make_function_from_raw_code(rc, context, args); } else { // default positional and keyword args not given - ffun = mp_make_function_from_raw_code(rc, MP_OBJ_NULL, MP_OBJ_NULL); + ffun = mp_make_function_from_raw_code(rc, context, NULL); } // wrap function in closure object return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2)); diff --git a/py/emitglue.h b/py/emitglue.h index a5411dc2e2..9a22a93218 100644 --- a/py/emitglue.h +++ b/py/emitglue.h @@ -54,16 +54,20 @@ typedef struct _mp_qstr_link_entry_t { uint16_t qst; } mp_qstr_link_entry_t; +// compiled bytecode: instance in RAM, referenced by outer scope, usually freed after first (and only) use +// mpy file: instance in RAM, created when .mpy file is loaded (same comments as above) +// frozen: instance in ROM typedef struct _mp_raw_code_t { mp_uint_t kind : 3; // of type mp_raw_code_kind_t mp_uint_t scope_flags : 7; mp_uint_t n_pos_args : 11; const void *fun_data; - const mp_uint_t *const_table; + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t fun_data_len; // so mp_raw_code_save and mp_bytecode_print work + #endif + struct _mp_raw_code_t **children; #if MICROPY_PERSISTENT_CODE_SAVE - size_t fun_data_len; - uint16_t n_obj; - uint16_t n_raw_code; + size_t n_children; #if MICROPY_PY_SYS_SETTRACE mp_bytecode_prelude_t prelude; // line_of_definition is a Python source line where the raw_code was @@ -89,22 +93,22 @@ void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS size_t len, #endif - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE - uint16_t n_obj, uint16_t n_raw_code, + size_t n_children, #endif mp_uint_t scope_flags); void mp_emit_glue_assign_native(mp_raw_code_t *rc, mp_raw_code_kind_t kind, void *fun_data, mp_uint_t fun_len, - const mp_uint_t *const_table, + mp_raw_code_t **children, #if MICROPY_PERSISTENT_CODE_SAVE + size_t n_children, uint16_t prelude_offset, - uint16_t n_obj, uint16_t n_raw_code, uint16_t n_qstr, mp_qstr_link_entry_t *qstr_link, #endif - mp_uint_t n_pos_args, mp_uint_t scope_flags, mp_uint_t type_sig); + mp_uint_t scope_flags, mp_uint_t n_pos_args, mp_uint_t type_sig); -mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args); -mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args); +mp_obj_t mp_make_function_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, const mp_obj_t *def_args); +mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, const mp_module_context_t *context, mp_uint_t n_closed_over, const mp_obj_t *args); #endif // MICROPY_INCLUDED_PY_EMITGLUE_H diff --git a/py/emitnative.c b/py/emitnative.c index 6504f37765..ca34e89f64 100644 --- a/py/emitnative.c +++ b/py/emitnative.c @@ -48,6 +48,7 @@ #include "py/emit.h" #include "py/nativeglue.h" +#include "py/objfun.h" #include "py/objstr.h" #if MICROPY_DEBUG_VERBOSE // print debugging info @@ -92,9 +93,13 @@ #define OFFSETOF_CODE_STATE_FUN_BC (offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)) #define OFFSETOF_CODE_STATE_IP (offsetof(mp_code_state_t, ip) / sizeof(uintptr_t)) #define OFFSETOF_CODE_STATE_SP (offsetof(mp_code_state_t, sp) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_GLOBALS (offsetof(mp_obj_fun_bc_t, globals) / sizeof(uintptr_t)) +#define OFFSETOF_CODE_STATE_N_STATE (offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_CONTEXT (offsetof(mp_obj_fun_bc_t, context) / sizeof(uintptr_t)) +#define OFFSETOF_OBJ_FUN_BC_CHILD_TABLE (offsetof(mp_obj_fun_bc_t, child_table) / sizeof(uintptr_t)) #define OFFSETOF_OBJ_FUN_BC_BYTECODE (offsetof(mp_obj_fun_bc_t, bytecode) / sizeof(uintptr_t)) -#define OFFSETOF_OBJ_FUN_BC_CONST_TABLE (offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_OBJ_TABLE (offsetof(mp_module_context_t, constants.obj_table) / sizeof(uintptr_t)) +#define OFFSETOF_MODULE_CONTEXT_GLOBALS (offsetof(mp_module_context_t, module.globals) / sizeof(uintptr_t)) +#define INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE (0) // If not already defined, set parent args to same as child call registers #ifndef REG_PARENT_RET @@ -205,6 +210,7 @@ typedef struct _exc_stack_entry_t { } exc_stack_entry_t; struct _emit_t { + mp_emit_common_t *emit_common; mp_obj_t *error_slot; uint *label_slot; uint exit_label; @@ -225,18 +231,17 @@ struct _emit_t { exc_stack_entry_t *exc_stack; int prelude_offset; + #if N_PRELUDE_AS_BYTES_OBJ + size_t prelude_const_table_offset; + #endif int start_offset; int n_state; uint16_t code_state_start; uint16_t stack_start; int stack_size; + uint16_t n_info; uint16_t n_cell; - uint16_t const_table_cur_obj; - uint16_t const_table_num_obj; - uint16_t const_table_cur_raw_code; - mp_uint_t *const_table; - #if MICROPY_PERSISTENT_CODE_SAVE uint16_t qstr_link_cur; mp_qstr_link_entry_t *qstr_link; @@ -255,8 +260,9 @@ STATIC void emit_native_global_exc_entry(emit_t *emit); STATIC void emit_native_global_exc_exit(emit_t *emit); STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj); -emit_t *EXPORT_FUN(new)(mp_obj_t * error_slot, uint *label_slot, mp_uint_t max_num_labels) { +emit_t *EXPORT_FUN(new)(mp_emit_common_t * emit_common, mp_obj_t *error_slot, uint *label_slot, mp_uint_t max_num_labels) { emit_t *emit = m_new0(emit_t, 1); + emit->emit_common = emit_common; emit->error_slot = error_slot; emit->label_slot = label_slot; emit->stack_info_alloc = 8; @@ -340,30 +346,22 @@ STATIC void emit_native_mov_reg_qstr_obj(emit_t *emit, int reg_dest, qstr qst) { emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ } while (false) -#define emit_native_mov_state_imm_fix_u16_via(emit, local_num, imm, reg_temp) \ - do { \ - ASM_MOV_REG_IMM_FIX_U16((emit)->as, (reg_temp), (imm)); \ - emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ - } while (false) - -#define emit_native_mov_state_imm_fix_word_via(emit, local_num, imm, reg_temp) \ - do { \ - ASM_MOV_REG_IMM_FIX_WORD((emit)->as, (reg_temp), (imm)); \ - emit_native_mov_state_reg((emit), (local_num), (reg_temp)); \ - } while (false) - STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { DEBUG_printf("start_pass(pass=%u, scope=%p)\n", pass, scope); + if (pass == MP_PASS_SCOPE) { + // Note: the first argument passed here is mp_emit_common_t, not the native emitter context + #if N_PRELUDE_AS_BYTES_OBJ + if (scope->emit_options == MP_EMIT_OPT_NATIVE_PYTHON) { + mp_emit_common_alloc_const_obj((mp_emit_common_t *)emit, mp_const_none); + } + #endif + return; + } + emit->pass = pass; emit->do_viper_types = scope->emit_options == MP_EMIT_OPT_VIPER; emit->stack_size = 0; - #if N_PRELUDE_AS_BYTES_OBJ - emit->const_table_cur_obj = emit->do_viper_types ? 0 : 1; // reserve first obj for prelude bytes obj - #else - emit->const_table_cur_obj = 0; - #endif - emit->const_table_cur_raw_code = 0; #if MICROPY_PERSISTENT_CODE_SAVE emit->qstr_link_cur = 0; #endif @@ -455,8 +453,9 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, 0); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE); // Store function object (passed as first arg) to stack if needed if (NEED_FUN_OBJ(emit)) { @@ -514,7 +513,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->stack_start = SIZEOF_CODE_STATE; #if N_PRELUDE_AS_BYTES_OBJ // Load index of prelude bytes object in const_table - mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)(emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1)); + mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_const_table_offset); #else mp_asm_base_data(&emit->as->base, ASM_WORD_SIZE, (uintptr_t)emit->prelude_offset); #endif @@ -536,8 +535,9 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_GENERATOR_STATE, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, emit->scope->num_pos_args + emit->scope->num_kwonly_args); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_TEMP0, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE); } else { // The locals and stack start after the code_state structure emit->stack_start = emit->code_state_start + SIZEOF_CODE_STATE; @@ -555,22 +555,33 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // Load REG_FUN_TABLE with a pointer to mp_fun_table, found in the const_table - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_FUN_TABLE, INDEX_OF_MP_FUN_TABLE_IN_CONST_TABLE); // Set code_state.fun_bc ASM_MOV_LOCAL_REG(emit->as, LOCAL_IDX_FUN_OBJ(emit), REG_PARENT_ARG_1); - // Set code_state.ip (offset from start of this function to prelude info) + // Set code_state.ip, a pointer to the beginning of the prelude + // Need to use some locals for this, so assert that they are available for use + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_1); + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_2); + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_3); + MP_STATIC_ASSERT(REG_LOCAL_3 != REG_PARENT_ARG_4); int code_state_ip_local = emit->code_state_start + OFFSETOF_CODE_STATE_IP; #if N_PRELUDE_AS_BYTES_OBJ - // Prelude is a bytes object in const_table; store ip = prelude->data - fun_bc->bytecode - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1); + // Prelude is a bytes object in const_table[prelude_const_table_offset]. + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, emit->prelude_const_table_offset); ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_LOCAL_3, offsetof(mp_obj_str_t, data) / sizeof(uintptr_t)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_PARENT_ARG_1, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE); - ASM_SUB_REG_REG(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1); - emit_native_mov_state_reg(emit, code_state_ip_local, REG_LOCAL_3); #else + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_1); + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_2); + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_3); + MP_STATIC_ASSERT(REG_LOCAL_2 != REG_PARENT_ARG_4); + // Prelude is at the end of the machine code + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_3, REG_PARENT_ARG_1, OFFSETOF_OBJ_FUN_BC_BYTECODE); if (emit->pass == MP_PASS_CODE_SIZE) { // Commit to the encoding size based on the value of prelude_offset in this pass. // By using 32768 as the cut-off it is highly unlikely that prelude_offset will @@ -579,14 +590,16 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop } if (emit->prelude_offset_uses_u16_encoding) { assert(emit->prelude_offset <= 65535); - emit_native_mov_state_imm_fix_u16_via(emit, code_state_ip_local, emit->prelude_offset, REG_PARENT_ARG_1); + ASM_MOV_REG_IMM_FIX_U16((emit)->as, REG_LOCAL_2, emit->prelude_offset); } else { - emit_native_mov_state_imm_fix_word_via(emit, code_state_ip_local, emit->prelude_offset, REG_PARENT_ARG_1); + ASM_MOV_REG_IMM_FIX_WORD((emit)->as, REG_LOCAL_2, emit->prelude_offset); } + ASM_ADD_REG_REG(emit->as, REG_LOCAL_3, REG_LOCAL_2); #endif + emit_native_mov_state_reg(emit, code_state_ip_local, REG_LOCAL_3); // Set code_state.n_state (only works on little endian targets due to n_state being uint16_t) - emit_native_mov_state_imm_via(emit, emit->code_state_start + offsetof(mp_code_state_t, n_state) / sizeof(uintptr_t), emit->n_state, REG_ARG_1); + emit_native_mov_state_imm_via(emit, emit->code_state_start + OFFSETOF_CODE_STATE_N_STATE, emit->n_state, REG_ARG_1); // Put address of code_state into first arg ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, emit->code_state_start); @@ -628,30 +641,17 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop emit->local_vtype[id->local_num] = VTYPE_PYOBJ; } } - - if (pass == MP_PASS_EMIT) { - // write argument names as qstr objects - // see comment in corresponding part of emitbc.c about the logic here - for (int i = 0; i < scope->num_pos_args + scope->num_kwonly_args; i++) { - qstr qst = MP_QSTR__star_; - for (int j = 0; j < scope->id_info_len; ++j) { - id_info_t *id = &scope->id_info[j]; - if ((id->flags & ID_FLAG_IS_PARAM) && id->local_num == i) { - qst = id->qst; - break; - } - } - emit->const_table[i] = (mp_uint_t)MP_OBJ_NEW_QSTR(qst); - } - } } - } static inline void emit_native_write_code_info_byte(emit_t *emit, byte val) { mp_asm_base_data(&emit->as->base, 1, val); } +static inline void emit_native_write_code_info_qstr(emit_t *emit, qstr qst) { + mp_encode_uint(&emit->as->base, mp_asm_base_get_cur_to_write_bytes, mp_emit_common_use_qstr(emit->emit_common, qst)); +} + STATIC void emit_native_end_pass(emit_t *emit) { emit_native_global_exc_exit(emit); @@ -662,21 +662,25 @@ STATIC void emit_native_end_pass(emit_t *emit) { size_t n_exc_stack = 0; // exc-stack not needed for native code MP_BC_PRELUDE_SIG_ENCODE(n_state, n_exc_stack, emit->scope, emit_native_write_code_info_byte, emit); - #if MICROPY_PERSISTENT_CODE - size_t n_info = 4; - #else - size_t n_info = 1; - #endif - MP_BC_PRELUDE_SIZE_ENCODE(n_info, emit->n_cell, emit_native_write_code_info_byte, emit); - - #if MICROPY_PERSISTENT_CODE - mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name); - mp_asm_base_data(&emit->as->base, 1, emit->scope->simple_name >> 8); - mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file); - mp_asm_base_data(&emit->as->base, 1, emit->scope->source_file >> 8); - #else - mp_asm_base_data(&emit->as->base, 1, 1); - #endif + size_t n_info = emit->n_info; + size_t n_cell = emit->n_cell; + MP_BC_PRELUDE_SIZE_ENCODE(n_info, n_cell, emit_native_write_code_info_byte, emit); + + // bytecode prelude: source info (function and argument qstrs) + size_t info_start = mp_asm_base_get_code_pos(&emit->as->base); + emit_native_write_code_info_qstr(emit, emit->scope->simple_name); + for (int i = 0; i < emit->scope->num_pos_args + emit->scope->num_kwonly_args; i++) { + qstr qst = MP_QSTR__star_; + for (int j = 0; j < emit->scope->id_info_len; ++j) { + id_info_t *id = &emit->scope->id_info[j]; + if ((id->flags & ID_FLAG_IS_PARAM) && id->local_num == i) { + qst = id->qst; + break; + } + } + emit_native_write_code_info_qstr(emit, qst); + } + emit->n_info = mp_asm_base_get_code_pos(&emit->as->base) - info_start; // bytecode prelude: initialise closed over variables size_t cell_start = mp_asm_base_get_code_pos(&emit->as->base); @@ -690,13 +694,14 @@ STATIC void emit_native_end_pass(emit_t *emit) { emit->n_cell = mp_asm_base_get_code_pos(&emit->as->base) - cell_start; #if N_PRELUDE_AS_BYTES_OBJ - // Prelude bytes object is after qstr arg names and mp_fun_table - size_t table_off = emit->scope->num_pos_args + emit->scope->num_kwonly_args + 1; + // Create the prelude as a bytes object, and store it in the constant table + mp_obj_t prelude = mp_const_none; if (emit->pass == MP_PASS_EMIT) { void *buf = emit->as->base.code_base + emit->prelude_offset; size_t n = emit->as->base.code_offset - emit->prelude_offset; - emit->const_table[table_off] = (uintptr_t)mp_obj_new_bytes(buf, n); + prelude = mp_obj_new_bytes(buf, n); } + emit->prelude_const_table_offset = mp_emit_common_alloc_const_obj(emit->emit_common, prelude); #endif } @@ -706,31 +711,15 @@ STATIC void emit_native_end_pass(emit_t *emit) { assert(emit->stack_size == 0); assert(emit->exc_stack_size == 0); - // Deal with const table accounting - assert(emit->pass <= MP_PASS_STACK_SIZE || (emit->const_table_num_obj == emit->const_table_cur_obj)); - emit->const_table_num_obj = emit->const_table_cur_obj; + #if MICROPY_PERSISTENT_CODE_SAVE + // Allocate qstr_link table if needed if (emit->pass == MP_PASS_CODE_SIZE) { - size_t const_table_alloc = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code; - size_t nqstr = 0; - if (!emit->do_viper_types) { - // Add room for qstr names of arguments - nqstr = emit->scope->num_pos_args + emit->scope->num_kwonly_args; - const_table_alloc += nqstr; - } - emit->const_table = m_new(mp_uint_t, const_table_alloc); - #if !MICROPY_DYNAMIC_COMPILER - // Store mp_fun_table pointer just after qstrs - // (but in dynamic-compiler mode eliminate dependency on mp_fun_table) - emit->const_table[nqstr] = (mp_uint_t)(uintptr_t)&mp_fun_table; - #endif - - #if MICROPY_PERSISTENT_CODE_SAVE size_t qstr_link_alloc = emit->qstr_link_cur; if (qstr_link_alloc > 0) { emit->qstr_link = m_new(mp_qstr_link_entry_t, qstr_link_alloc); } - #endif } + #endif if (emit->pass == MP_PASS_EMIT) { void *f = mp_asm_base_get_code(&emit->as->base); @@ -738,13 +727,14 @@ STATIC void emit_native_end_pass(emit_t *emit) { mp_emit_glue_assign_native(emit->scope->raw_code, emit->do_viper_types ? MP_CODE_NATIVE_VIPER : MP_CODE_NATIVE_PY, - f, f_len, emit->const_table, + f, f_len, + emit->emit_common->children, #if MICROPY_PERSISTENT_CODE_SAVE + emit->emit_common->ct_cur_child, emit->prelude_offset, - emit->const_table_cur_obj, emit->const_table_cur_raw_code, emit->qstr_link_cur, emit->qstr_link, #endif - emit->scope->num_pos_args, emit->scope->scope_flags, 0); + emit->scope->scope_flags, 0, 0); } } @@ -1137,29 +1127,19 @@ STATIC exc_stack_entry_t *emit_native_pop_exc_stack(emit_t *emit) { return e; } -STATIC void emit_load_reg_with_ptr(emit_t *emit, int reg, mp_uint_t ptr, size_t table_off) { - if (!emit->do_viper_types) { - // Skip qstr names of arguments - table_off += emit->scope->num_pos_args + emit->scope->num_kwonly_args; - } - if (emit->pass == MP_PASS_EMIT) { - emit->const_table[table_off] = ptr; - } +STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) { + size_t table_off = mp_emit_common_alloc_const_obj(emit->emit_common, obj); emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONST_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off); } -STATIC void emit_load_reg_with_object(emit_t *emit, int reg, mp_obj_t obj) { - // First entry is for mp_fun_table - size_t table_off = 1 + emit->const_table_cur_obj++; - emit_load_reg_with_ptr(emit, reg, (mp_uint_t)obj, table_off); -} - -STATIC void emit_load_reg_with_raw_code(emit_t *emit, int reg, mp_raw_code_t *rc) { - // First entry is for mp_fun_table, then constant objects - size_t table_off = 1 + emit->const_table_num_obj + emit->const_table_cur_raw_code++; - emit_load_reg_with_ptr(emit, reg, (mp_uint_t)rc, table_off); +STATIC void emit_load_reg_with_child(emit_t *emit, int reg, mp_raw_code_t *rc) { + size_t table_off = mp_emit_common_alloc_const_child(emit->emit_common, rc); + emit_native_mov_reg_state(emit, REG_TEMP0, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_TEMP0, REG_TEMP0, OFFSETOF_OBJ_FUN_BC_CHILD_TABLE); + ASM_LOAD_REG_REG_OFFSET(emit->as, reg, REG_TEMP0, table_off); } STATIC void emit_native_label_assign(emit_t *emit, mp_uint_t l) { @@ -1203,7 +1183,8 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { if (!(emit->scope->scope_flags & MP_SCOPE_FLAG_GENERATOR)) { // Set new globals emit_native_mov_reg_state(emit, REG_ARG_1, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_GLOBALS); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_1, REG_ARG_1, OFFSETOF_MODULE_CONTEXT_GLOBALS); emit_call(emit, MP_F_NATIVE_SWAP_GLOBALS); // Save old globals (or NULL if globals didn't change) @@ -1254,7 +1235,8 @@ STATIC void emit_native_global_exc_entry(emit_t *emit) { #if N_NLR_SETJMP // Reload REG_FUN_TABLE, since it may be clobbered by longjmp emit_native_mov_reg_state(emit, REG_LOCAL_1, LOCAL_IDX_FUN_OBJ(emit)); - ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, offsetof(mp_obj_fun_bc_t, const_table) / sizeof(uintptr_t)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, OFFSETOF_OBJ_FUN_BC_CONTEXT); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_LOCAL_1, REG_LOCAL_1, OFFSETOF_MODULE_CONTEXT_OBJ_TABLE); ASM_LOAD_REG_REG_OFFSET(emit->as, REG_FUN_TABLE, REG_LOCAL_1, emit->scope->num_pos_args + emit->scope->num_kwonly_args); #endif ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, LOCAL_IDX_EXC_HANDLER_PC(emit)); @@ -1385,11 +1367,7 @@ STATIC void emit_native_import(emit_t *emit, qstr qst, int kind) { STATIC void emit_native_load_const_tok(emit_t *emit, mp_token_kind_t tok) { DEBUG_printf("load_const_tok(tok=%u)\n", tok); if (tok == MP_TOKEN_ELLIPSIS) { - #if MICROPY_PERSISTENT_CODE_SAVE emit_native_load_const_obj(emit, MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj)); - #else - emit_post_push_imm(emit, VTYPE_PYOBJ, (mp_uint_t)MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj)); - #endif } else { emit_native_pre(emit); if (tok == MP_TOKEN_KW_NONE) { @@ -2682,33 +2660,46 @@ STATIC void emit_native_unpack_ex(emit_t *emit, mp_uint_t n_left, mp_uint_t n_ri STATIC void emit_native_make_function(emit_t *emit, scope_t *scope, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { // call runtime, with type info for args, or don't support dict/default params, or only support Python objects for them emit_native_pre(emit); + emit_native_mov_reg_state(emit, REG_ARG_2, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_2, REG_ARG_2, OFFSETOF_OBJ_FUN_BC_CONTEXT); if (n_pos_defaults == 0 && n_kw_defaults == 0) { need_reg_all(emit); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)MP_OBJ_NULL); - ASM_MOV_REG_IMM(emit->as, REG_ARG_3, (mp_uint_t)MP_OBJ_NULL); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, 0); } else { - vtype_kind_t vtype_def_tuple, vtype_def_dict; - emit_pre_pop_reg_reg(emit, &vtype_def_dict, REG_ARG_3, &vtype_def_tuple, REG_ARG_2); - assert(vtype_def_tuple == VTYPE_PYOBJ); - assert(vtype_def_dict == VTYPE_PYOBJ); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, 2); need_reg_all(emit); } - emit_load_reg_with_raw_code(emit, REG_ARG_1, scope->raw_code); + emit_load_reg_with_child(emit, REG_ARG_1, scope->raw_code); ASM_CALL_IND(emit->as, MP_F_MAKE_FUNCTION_FROM_RAW_CODE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } STATIC void emit_native_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_closed_over, mp_uint_t n_pos_defaults, mp_uint_t n_kw_defaults) { + // make function emit_native_pre(emit); + emit_native_mov_reg_state(emit, REG_ARG_2, LOCAL_IDX_FUN_OBJ(emit)); + ASM_LOAD_REG_REG_OFFSET(emit->as, REG_ARG_2, REG_ARG_2, OFFSETOF_OBJ_FUN_BC_CONTEXT); if (n_pos_defaults == 0 && n_kw_defaults == 0) { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); + need_reg_all(emit); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, 0); } else { - emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over + 2); - ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0x100 | n_closed_over); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, 2 + n_closed_over); + adjust_stack(emit, 2 + n_closed_over); + need_reg_all(emit); + } + emit_load_reg_with_child(emit, REG_ARG_1, scope->raw_code); + ASM_CALL_IND(emit->as, MP_F_MAKE_FUNCTION_FROM_RAW_CODE); + + // make closure + #if REG_ARG_1 != REG_RET + ASM_MOV_REG_REG(emit->as, REG_ARG_1, REG_RET); + #endif + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); + emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); + if (n_pos_defaults != 0 || n_kw_defaults != 0) { + adjust_stack(emit, -2); } - emit_load_reg_with_raw_code(emit, REG_ARG_1, scope->raw_code); - ASM_CALL_IND(emit->as, MP_F_MAKE_CLOSURE_FROM_RAW_CODE); + ASM_CALL_IND(emit->as, MP_F_NEW_CLOSURE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } diff --git a/py/emitnx86.c b/py/emitnx86.c index f0553f0682..d990ef5a9e 100644 --- a/py/emitnx86.c +++ b/py/emitnx86.c @@ -56,7 +56,7 @@ STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { [MP_F_UNPACK_EX] = 3, [MP_F_DELETE_NAME] = 1, [MP_F_DELETE_GLOBAL] = 1, - [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, + [MP_F_NEW_CLOSURE] = 3, [MP_F_ARG_CHECK_NUM_SIG] = 3, [MP_F_SETUP_CODE_STATE] = 4, [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, diff --git a/py/frozenmod.c b/py/frozenmod.c index 6cb68d1ec0..61c2f20aa1 100644 --- a/py/frozenmod.c +++ b/py/frozenmod.c @@ -57,7 +57,7 @@ extern const char mp_frozen_str_content[]; #include "py/emitglue.h" -extern const mp_raw_code_t *const mp_frozen_mpy_content[]; +extern const mp_frozen_module_t *const mp_frozen_mpy_content[]; #endif // MICROPY_MODULE_FROZEN_MPY diff --git a/py/mpconfig.h b/py/mpconfig.h index 47779a67de..5edff69dfd 100644 --- a/py/mpconfig.h +++ b/py/mpconfig.h @@ -328,6 +328,14 @@ #define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY) #endif +// Whether bytecode uses a qstr_table to map internal qstr indices in the bytecode +// to global qstr values in the runtime (behaviour when feature is enabled), or +// just stores global qstr values directly in the bytecode. This must be enabled +// if MICROPY_PERSISTENT_CODE is enabled. +#ifndef MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE +#define MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE (MICROPY_PERSISTENT_CODE) +#endif + // Whether to emit x64 native code #ifndef MICROPY_EMIT_X64 #define MICROPY_EMIT_X64 (0) diff --git a/py/nativeglue.c b/py/nativeglue.c index 30e5b40061..aed6fecddf 100644 --- a/py/nativeglue.c +++ b/py/nativeglue.c @@ -300,7 +300,7 @@ const mp_fun_table_t mp_fun_table = { mp_unpack_ex, mp_delete_name, mp_delete_global, - mp_make_closure_from_raw_code, + mp_obj_new_closure, mp_arg_check_num_sig, mp_setup_code_state, mp_small_int_floor_divide, @@ -344,4 +344,8 @@ const mp_fun_table_t mp_fun_table = { &mp_stream_write_obj, }; +#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER + +const int mp_fun_table; + #endif // MICROPY_EMIT_NATIVE diff --git a/py/nativeglue.h b/py/nativeglue.h index 9d9a97b9e7..49ce665f2d 100644 --- a/py/nativeglue.h +++ b/py/nativeglue.h @@ -75,7 +75,7 @@ typedef enum { MP_F_UNPACK_EX, MP_F_DELETE_NAME, MP_F_DELETE_GLOBAL, - MP_F_MAKE_CLOSURE_FROM_RAW_CODE, + MP_F_NEW_CLOSURE, MP_F_ARG_CHECK_NUM_SIG, MP_F_SETUP_CODE_STATE, MP_F_SMALL_INT_FLOOR_DIVIDE, @@ -112,7 +112,7 @@ typedef struct _mp_fun_table_t { void (*set_store)(mp_obj_t self_in, mp_obj_t item); mp_obj_t (*list_append)(mp_obj_t self_in, mp_obj_t arg); mp_obj_t (*dict_store)(mp_obj_t self_in, mp_obj_t key, mp_obj_t value); - mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, mp_obj_t def_args, mp_obj_t def_kw_args); + mp_obj_t (*make_function_from_raw_code)(const mp_raw_code_t *rc, const mp_module_context_t *cm, const mp_obj_t *def_args); mp_obj_t (*call_function_n_kw)(mp_obj_t fun_in, size_t n_args_kw, const mp_obj_t *args); mp_obj_t (*call_method_n_kw)(size_t n_args, size_t n_kw, const mp_obj_t *args); mp_obj_t (*call_method_n_kw_var)(bool have_self, size_t n_args_n_kw, const mp_obj_t *args); @@ -129,7 +129,7 @@ typedef struct _mp_fun_table_t { void (*unpack_ex)(mp_obj_t seq, size_t num, mp_obj_t *items); void (*delete_name)(qstr qst); void (*delete_global)(qstr qst); - mp_obj_t (*make_closure_from_raw_code)(const mp_raw_code_t *rc, mp_uint_t n_closed_over, const mp_obj_t *args); + mp_obj_t (*new_closure)(mp_obj_t fun, size_t n_closed_over, const mp_obj_t *closed); void (*arg_check_num_sig)(size_t n_args, size_t n_kw, uint32_t sig); void (*setup_code_state)(mp_code_state_t *code_state, size_t n_args, size_t n_kw, const mp_obj_t *args); mp_int_t (*small_int_floor_divide)(mp_int_t num, mp_int_t denom); @@ -172,6 +172,12 @@ typedef struct _mp_fun_table_t { const mp_obj_fun_builtin_var_t *stream_write_obj; } mp_fun_table_t; +#if (MICROPY_EMIT_NATIVE && !MICROPY_DYNAMIC_COMPILER) || MICROPY_ENABLE_DYNRUNTIME extern const mp_fun_table_t mp_fun_table; +#elif MICROPY_EMIT_NATIVE && MICROPY_DYNAMIC_COMPILER +// In dynamic-compiler mode eliminate dependency on entries in mp_fun_table. +// This only needs to be an independent pointer, content doesn't matter. +extern const int mp_fun_table; +#endif #endif // MICROPY_INCLUDED_PY_NATIVEGLUE_H @@ -781,9 +781,6 @@ mp_obj_t mp_obj_new_exception_msg_varg(const mp_obj_type_t *exc_type, mp_rom_err #ifdef va_start mp_obj_t mp_obj_new_exception_msg_vlist(const mp_obj_type_t *exc_type, mp_rom_error_text_t fmt, va_list arg); // same fmt restrictions as above #endif -mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table); -mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table); -mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig); mp_obj_t mp_obj_new_gen_wrap(mp_obj_t fun); mp_obj_t mp_obj_new_closure(mp_obj_t fun, size_t n_closed, const mp_obj_t *closed); mp_obj_t mp_obj_new_tuple(size_t n, const mp_obj_t *items); @@ -992,7 +989,6 @@ typedef struct _mp_obj_fun_builtin_var_t { } mp_obj_fun_builtin_var_t; qstr mp_obj_fun_get_name(mp_const_obj_t fun); -qstr mp_obj_code_get_name(const byte *code_info); mp_obj_t mp_identity(mp_obj_t self); MP_DECLARE_CONST_FUN_OBJ_1(mp_identity_obj); diff --git a/py/objfun.c b/py/objfun.c index 85f531c88b..3542cc0e3f 100644 --- a/py/objfun.c +++ b/py/objfun.c @@ -143,13 +143,13 @@ const mp_obj_type_t mp_type_fun_builtin_var = { /******************************************************************************/ /* byte code functions */ -qstr mp_obj_code_get_name(const byte *code_info) { +STATIC qstr mp_obj_code_get_name(const mp_obj_fun_bc_t *fun, const byte *code_info) { MP_BC_PRELUDE_SIZE_DECODE(code_info); - #if MICROPY_PERSISTENT_CODE - return code_info[0] | (code_info[1] << 8); - #else - return mp_decode_uint_value(code_info); + mp_uint_t name = mp_decode_uint_value(code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + name = fun->context->constants.qstr_table[name]; #endif + return name; } #if MICROPY_EMIT_NATIVE @@ -167,7 +167,7 @@ qstr mp_obj_fun_get_name(mp_const_obj_t fun_in) { const byte *bc = fun->bytecode; MP_BC_PRELUDE_SIG_DECODE(bc); - return mp_obj_code_get_name(bc); + return mp_obj_code_get_name(fun, bc); } #if MICROPY_CPYTHON_COMPAT @@ -209,7 +209,7 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { #define INIT_CODESTATE(code_state, _fun_bc, _n_state, n_args, n_kw, args) \ code_state->fun_bc = _fun_bc; \ - code_state->ip = 0; \ + code_state->ip = _fun_bc->bytecode; \ code_state->n_state = _n_state; \ mp_setup_code_state(code_state, n_args, n_kw, args); \ code_state->old_globals = mp_globals_get(); @@ -240,7 +240,7 @@ mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(self->globals); + mp_globals_set(self->context->module.globals); return code_state; } @@ -285,7 +285,7 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const INIT_CODESTATE(code_state, self, n_state, n_args, n_kw, args); // execute the byte code with the correct globals context - mp_globals_set(self->globals); + mp_globals_set(self->context->module.globals); mp_vm_return_kind_t vm_return_kind = mp_execute_bytecode(code_state, MP_OBJ_NULL); mp_globals_set(code_state->old_globals); @@ -358,7 +358,7 @@ void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } if (attr == MP_QSTR___globals__) { mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - dest[0] = MP_OBJ_FROM_PTR(self->globals); + dest[0] = MP_OBJ_FROM_PTR(self->context->module.globals); } } #endif @@ -377,25 +377,29 @@ const mp_obj_type_t mp_type_fun_bc = { #endif }; -mp_obj_t mp_obj_new_fun_bc(mp_obj_t def_args_in, mp_obj_t def_kw_args, const byte *code, const mp_uint_t *const_table) { +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *context, struct _mp_raw_code_t *const *child_table) { size_t n_def_args = 0; size_t n_extra_args = 0; - mp_obj_tuple_t *def_args = MP_OBJ_TO_PTR(def_args_in); - if (def_args_in != MP_OBJ_NULL) { - assert(mp_obj_is_type(def_args_in, &mp_type_tuple)); - n_def_args = def_args->len; - n_extra_args = def_args->len; + mp_obj_tuple_t *def_pos_args = NULL; + mp_obj_t def_kw_args = MP_OBJ_NULL; + if (def_args != NULL && def_args[0] != MP_OBJ_NULL) { + assert(mp_obj_is_type(def_args[0], &mp_type_tuple)); + def_pos_args = MP_OBJ_TO_PTR(def_args[0]); + n_def_args = def_pos_args->len; + n_extra_args = def_pos_args->len; } - if (def_kw_args != MP_OBJ_NULL) { + if (def_args != NULL && def_args[1] != MP_OBJ_NULL) { + assert(mp_obj_is_type(def_args[1], &mp_type_dict)); + def_kw_args = def_args[1]; n_extra_args += 1; } mp_obj_fun_bc_t *o = m_new_obj_var(mp_obj_fun_bc_t, mp_obj_t, n_extra_args); o->base.type = &mp_type_fun_bc; - o->globals = mp_globals_get(); o->bytecode = code; - o->const_table = const_table; - if (def_args != NULL) { - memcpy(o->extra_args, def_args->items, n_def_args * sizeof(mp_obj_t)); + o->context = context; + o->child_table = child_table; + if (def_pos_args != NULL) { + memcpy(o->extra_args, def_pos_args->items, n_def_args * sizeof(mp_obj_t)); } if (def_kw_args != MP_OBJ_NULL) { o->extra_args[n_def_args] = def_kw_args; @@ -423,8 +427,8 @@ STATIC const mp_obj_type_t mp_type_fun_native = { .unary_op = mp_generic_unary_op, }; -mp_obj_t mp_obj_new_fun_native(mp_obj_t def_args_in, mp_obj_t def_kw_args, const void *fun_data, const mp_uint_t *const_table) { - mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args_in, def_kw_args, (const byte *)fun_data, const_table)); +mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *mc, struct _mp_raw_code_t *const *child_table) { + mp_obj_fun_bc_t *o = MP_OBJ_TO_PTR(mp_obj_new_fun_bc(def_args, (const byte *)fun_data, mc, child_table)); o->base.type = &mp_type_fun_native; return MP_OBJ_FROM_PTR(o); } diff --git a/py/objfun.h b/py/objfun.h index 771bf31a95..9de15b8841 100644 --- a/py/objfun.h +++ b/py/objfun.h @@ -26,13 +26,14 @@ #ifndef MICROPY_INCLUDED_PY_OBJFUN_H #define MICROPY_INCLUDED_PY_OBJFUN_H +#include "py/bc.h" #include "py/obj.h" typedef struct _mp_obj_fun_bc_t { mp_obj_base_t base; - mp_obj_dict_t *globals; // the context within which this function was defined - const byte *bytecode; // bytecode for the function - const mp_uint_t *const_table; // constant table + const mp_module_context_t *context; // context within which this function was defined + struct _mp_raw_code_t *const *child_table; // table of children + const byte *bytecode; // bytecode for the function #if MICROPY_PY_SYS_SETTRACE const struct _mp_raw_code_t *rc; #endif @@ -42,6 +43,9 @@ typedef struct _mp_obj_fun_bc_t { mp_obj_t extra_args[]; } mp_obj_fun_bc_t; +mp_obj_t mp_obj_new_fun_bc(const mp_obj_t *def_args, const byte *code, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +mp_obj_t mp_obj_new_fun_native(const mp_obj_t *def_args, const void *fun_data, const mp_module_context_t *cm, struct _mp_raw_code_t *const *raw_code_table); +mp_obj_t mp_obj_new_fun_asm(size_t n_args, const void *fun_data, mp_uint_t type_sig); void mp_obj_fun_bc_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); #endif // MICROPY_INCLUDED_PY_OBJFUN_H diff --git a/py/objgenerator.c b/py/objgenerator.c index 784310092e..cbe79e66a5 100644 --- a/py/objgenerator.c +++ b/py/objgenerator.c @@ -65,7 +65,7 @@ STATIC mp_obj_t gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_kw, cons o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; - o->code_state.ip = 0; + o->code_state.ip = self_fun->bytecode; o->code_state.n_state = n_state; mp_setup_code_state(&o->code_state, n_args, n_kw, args); return MP_OBJ_FROM_PTR(o); @@ -91,14 +91,18 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k // The state for a native generating function is held in the same struct as a bytecode function mp_obj_fun_bc_t *self_fun = MP_OBJ_TO_PTR(self_in); - // Determine start of prelude, and extract n_state from it + // Determine start of prelude. uintptr_t prelude_offset = ((uintptr_t *)self_fun->bytecode)[0]; #if MICROPY_EMIT_NATIVE_PRELUDE_AS_BYTES_OBJ // Prelude is in bytes object in const_table, at index prelude_offset - mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->const_table[prelude_offset]); - prelude_offset = (const byte *)prelude_bytes->data - self_fun->bytecode; + mp_obj_str_t *prelude_bytes = MP_OBJ_TO_PTR(self_fun->context->constants.obj_table[prelude_offset]); + const uint8_t *prelude_ptr = prelude_bytes->data; + #else + const uint8_t *prelude_ptr = self_fun->bytecode + prelude_offset; #endif - const uint8_t *ip = self_fun->bytecode + prelude_offset; + + // Extract n_state from the prelude. + const uint8_t *ip = prelude_ptr; size_t n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args; MP_BC_PRELUDE_SIG_DECODE_INTO(ip, n_state, n_exc_stack_unused, scope_flags, n_pos_args, n_kwonly_args, n_def_args); size_t n_exc_stack = 0; @@ -111,7 +115,7 @@ STATIC mp_obj_t native_gen_wrap_call(mp_obj_t self_in, size_t n_args, size_t n_k // Parse the input arguments and set up the code state o->pend_exc = mp_const_none; o->code_state.fun_bc = self_fun; - o->code_state.ip = (const byte *)prelude_offset; + o->code_state.ip = prelude_ptr; o->code_state.n_state = n_state; mp_setup_code_state(&o->code_state, n_args, n_kw, args); @@ -184,7 +188,7 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ // Set up the correct globals context for the generator and execute it self->code_state.old_globals = mp_globals_get(); - mp_globals_set(self->code_state.fun_bc->globals); + mp_globals_set(self->code_state.fun_bc->context->module.globals); mp_vm_return_kind_t ret_kind; diff --git a/py/objmodule.c b/py/objmodule.c index 9be4bad92c..4b04f7ca9c 100644 --- a/py/objmodule.c +++ b/py/objmodule.c @@ -112,6 +112,7 @@ const mp_obj_type_t mp_type_module = { .attr = module_attr, }; +#include "py/bc.h" mp_obj_t mp_obj_new_module(qstr module_name) { mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; mp_map_elem_t *el = mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(module_name), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); @@ -122,12 +123,12 @@ mp_obj_t mp_obj_new_module(qstr module_name) { } // create new module object - mp_obj_module_t *o = m_new_obj(mp_obj_module_t); - o->base.type = &mp_type_module; - o->globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); + mp_module_context_t *o = m_new_obj(mp_module_context_t); + o->module.base.type = &mp_type_module; + o->module.globals = MP_OBJ_TO_PTR(mp_obj_new_dict(MICROPY_MODULE_DICT_SIZE)); // store __name__ entry in the module - mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); + mp_obj_dict_store(MP_OBJ_FROM_PTR(o->module.globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(module_name)); // store the new module into the slot in the global dict holding all modules el->value = MP_OBJ_FROM_PTR(o); diff --git a/py/persistentcode.c b/py/persistentcode.c index ac523990c1..b473f18308 100644 --- a/py/persistentcode.c +++ b/py/persistentcode.c @@ -63,57 +63,6 @@ STATIC int mp_small_int_bits(void) { } #endif -#define QSTR_WINDOW_SIZE (32) - -typedef struct _qstr_window_t { - uint16_t idx; // indexes the head of the window - uint16_t window[QSTR_WINDOW_SIZE]; -} qstr_window_t; - -// Push a qstr to the head of the window, and the tail qstr is overwritten -STATIC void qstr_window_push(qstr_window_t *qw, qstr qst) { - qw->idx = (qw->idx + 1) % QSTR_WINDOW_SIZE; - qw->window[qw->idx] = qst; -} - -// Pull an existing qstr from within the window to the head of the window -STATIC qstr qstr_window_pull(qstr_window_t *qw, size_t idx) { - qstr qst = qw->window[idx]; - if (idx > qw->idx) { - memmove(&qw->window[idx], &qw->window[idx + 1], (QSTR_WINDOW_SIZE - idx - 1) * sizeof(uint16_t)); - qw->window[QSTR_WINDOW_SIZE - 1] = qw->window[0]; - idx = 0; - } - memmove(&qw->window[idx], &qw->window[idx + 1], (qw->idx - idx) * sizeof(uint16_t)); - qw->window[qw->idx] = qst; - return qst; -} - -#if MICROPY_PERSISTENT_CODE_LOAD - -// Access a qstr at the given index, relative to the head of the window (0=head) -STATIC qstr qstr_window_access(qstr_window_t *qw, size_t idx) { - return qstr_window_pull(qw, (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE); -} - -#endif - -#if MICROPY_PERSISTENT_CODE_SAVE - -// Insert a qstr at the head of the window, either by pulling an existing one or pushing a new one -STATIC size_t qstr_window_insert(qstr_window_t *qw, qstr qst) { - for (size_t idx = 0; idx < QSTR_WINDOW_SIZE; ++idx) { - if (qw->window[idx] == qst) { - qstr_window_pull(qw, idx); - return (qw->idx + QSTR_WINDOW_SIZE - idx) % QSTR_WINDOW_SIZE; - } - } - qstr_window_push(qw, qst); - return QSTR_WINDOW_SIZE; -} - -#endif - typedef struct _bytecode_prelude_t { uint n_state; uint n_exc_stack; @@ -124,23 +73,6 @@ typedef struct _bytecode_prelude_t { uint code_info_size; } bytecode_prelude_t; -// ip will point to start of opcodes -// return value will point to simple_name, source_file qstrs -STATIC byte *extract_prelude(const byte **ip, bytecode_prelude_t *prelude) { - MP_BC_PRELUDE_SIG_DECODE(*ip); - prelude->n_state = n_state; - prelude->n_exc_stack = n_exc_stack; - prelude->scope_flags = scope_flags; - prelude->n_pos_args = n_pos_args; - prelude->n_kwonly_args = n_kwonly_args; - prelude->n_def_pos_args = n_def_pos_args; - MP_BC_PRELUDE_SIZE_DECODE(*ip); - byte *ip_info = (byte *)*ip; - *ip += n_info; - *ip += n_cell; - return ip_info; -} - #endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE #if MICROPY_PERSISTENT_CODE_LOAD @@ -148,13 +80,14 @@ STATIC byte *extract_prelude(const byte **ip, bytecode_prelude_t *prelude) { #include "py/parsenum.h" STATIC int read_byte(mp_reader_t *reader); -STATIC size_t read_uint(mp_reader_t *reader, byte **out); +STATIC size_t read_uint(mp_reader_t *reader); #if MICROPY_EMIT_MACHINE_CODE typedef struct _reloc_info_t { mp_reader_t *reader; - mp_uint_t *const_table; + uint8_t *rodata; + uint8_t *bss; } reloc_info_t; #if MICROPY_EMIT_THUMB @@ -197,13 +130,13 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { while ((op = read_byte(ri->reader)) != 0xff) { if (op & 1) { // Point to new location to make adjustments - size_t addr = read_uint(ri->reader, NULL); + size_t addr = read_uint(ri->reader); if ((addr & 1) == 0) { // Point to somewhere in text addr_to_adjust = &((uintptr_t *)text)[addr >> 1]; } else { // Point to somewhere in rodata - addr_to_adjust = &((uintptr_t *)ri->const_table[1])[addr >> 1]; + addr_to_adjust = &((uintptr_t *)ri->rodata)[addr >> 1]; } } op >>= 1; @@ -212,15 +145,18 @@ void mp_native_relocate(void *ri_in, uint8_t *text, uintptr_t reloc_text) { if (op <= 5) { if (op & 1) { // Read in number of adjustments to make - n = read_uint(ri->reader, NULL); + n = read_uint(ri->reader); } op >>= 1; if (op == 0) { // Destination is text dest = reloc_text; + } else if (op == 1) { + // Destination is rodata + dest = (uintptr_t)ri->rodata; } else { - // Destination is rodata (op=1) or bss (op=1 if no rodata, else op=2) - dest = ri->const_table[op]; + // Destination is bss + dest = (uintptr_t)ri->bss; } } else if (op == 6) { // Destination is mp_fun_table itself @@ -247,14 +183,10 @@ STATIC void read_bytes(mp_reader_t *reader, byte *buf, size_t len) { } } -STATIC size_t read_uint(mp_reader_t *reader, byte **out) { +STATIC size_t read_uint(mp_reader_t *reader) { size_t unum = 0; for (;;) { byte b = reader->readbyte(reader->data); - if (out != NULL) { - **out = b; - ++*out; - } unum = (unum << 7) | (b & 0x7f); if ((b & 0x80) == 0) { break; @@ -263,35 +195,41 @@ STATIC size_t read_uint(mp_reader_t *reader, byte **out) { return unum; } -STATIC qstr load_qstr(mp_reader_t *reader, qstr_window_t *qw) { - size_t len = read_uint(reader, NULL); - if (len == 0) { - // static qstr - return read_byte(reader); - } +STATIC qstr load_qstr(mp_reader_t *reader) { + size_t len = read_uint(reader); if (len & 1) { - // qstr in window - return qstr_window_access(qw, len >> 1); + // static qstr + return len >> 1; } len >>= 1; char *str = m_new(char, len); read_bytes(reader, (byte *)str, len); + read_byte(reader); // read and discard null terminator qstr qst = qstr_from_strn(str, len); m_del(char, str, len); - qstr_window_push(qw, qst); return qst; } STATIC mp_obj_t load_obj(mp_reader_t *reader) { byte obj_type = read_byte(reader); + #if MICROPY_EMIT_MACHINE_CODE + if (obj_type == 't') { + return MP_OBJ_FROM_PTR(&mp_fun_table); + } else + #endif if (obj_type == 'e') { return MP_OBJ_FROM_PTR(&mp_const_ellipsis_obj); } else { - size_t len = read_uint(reader, NULL); + size_t len = read_uint(reader); + if (len == 0 && obj_type == 'b') { + read_byte(reader); // skip null terminator + return mp_const_empty_bytes; + } vstr_t vstr; vstr_init_len(&vstr, len); read_bytes(reader, (byte *)vstr.buf, len); if (obj_type == 's' || obj_type == 'b') { + read_byte(reader); // skip null terminator return mp_obj_new_str_from_vstr(obj_type == 's' ? &mp_type_str : &mp_type_bytes, &vstr); } else if (obj_type == 'i') { return mp_parse_num_integer(vstr.buf, vstr.len, 10, NULL); @@ -302,58 +240,12 @@ STATIC mp_obj_t load_obj(mp_reader_t *reader) { } } -STATIC void load_prelude_qstrs(mp_reader_t *reader, qstr_window_t *qw, byte *ip) { - qstr simple_name = load_qstr(reader, qw); - ip[0] = simple_name; - ip[1] = simple_name >> 8; - qstr source_file = load_qstr(reader, qw); - ip[2] = source_file; - ip[3] = source_file >> 8; -} - -STATIC void load_prelude(mp_reader_t *reader, qstr_window_t *qw, byte **ip, bytecode_prelude_t *prelude) { - // Read in the prelude header - byte *ip_read = *ip; - read_uint(reader, &ip_read); // read in n_state/etc (is effectively a var-uint) - read_uint(reader, &ip_read); // read in n_info/n_cell (is effectively a var-uint) - - // Prelude header has been read into *ip, now decode and extract values from it - extract_prelude((const byte **)ip, prelude); - - // Load qstrs in prelude - load_prelude_qstrs(reader, qw, ip_read); - ip_read += 4; - - // Read remaining code info - read_bytes(reader, ip_read, *ip - ip_read); -} - -STATIC void load_bytecode(mp_reader_t *reader, qstr_window_t *qw, byte *ip, byte *ip_top) { - while (ip < ip_top) { - *ip = read_byte(reader); - size_t sz; - uint f = mp_opcode_format(ip, &sz, false); - ++ip; - --sz; - if (f == MP_BC_FORMAT_QSTR) { - qstr qst = load_qstr(reader, qw); - *ip++ = qst; - *ip++ = qst >> 8; - sz -= 2; - } else if (f == MP_BC_FORMAT_VAR_UINT) { - while ((*ip++ = read_byte(reader)) & 0x80) { - } - } - read_bytes(reader, ip, sz); - ip += sz; - } -} - -STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { +STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { // Load function kind and data length - size_t kind_len = read_uint(reader, NULL); + size_t kind_len = read_uint(reader); int kind = (kind_len & 3) + MP_CODE_BYTECODE; - size_t fun_data_len = kind_len >> 2; + bool has_children = !!(kind_len & 4); + size_t fun_data_len = kind_len >> 3; #if !MICROPY_EMIT_MACHINE_CODE if (kind != MP_CODE_BYTECODE) { @@ -362,23 +254,18 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { #endif uint8_t *fun_data = NULL; - bytecode_prelude_t prelude = {0}; #if MICROPY_EMIT_MACHINE_CODE size_t prelude_offset = 0; - mp_uint_t type_sig = 0; - size_t n_qstr_link = 0; + mp_uint_t native_scope_flags = 0; + mp_uint_t native_n_pos_args = 0; + mp_uint_t native_type_sig = 0; #endif if (kind == MP_CODE_BYTECODE) { // Allocate memory for the bytecode fun_data = m_new(uint8_t, fun_data_len); - - // Load prelude - byte *ip = fun_data; - load_prelude(reader, qw, &ip, &prelude); - // Load bytecode - load_bytecode(reader, qw, ip, fun_data + fun_data_len); + read_bytes(reader, fun_data, fun_data_len); #if MICROPY_EMIT_MACHINE_CODE } else { @@ -389,10 +276,10 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { if (kind == MP_CODE_NATIVE_PY || kind == MP_CODE_NATIVE_VIPER) { // Parse qstr link table and link native code - n_qstr_link = read_uint(reader, NULL); + size_t n_qstr_link = read_uint(reader); for (size_t i = 0; i < n_qstr_link; ++i) { - size_t off = read_uint(reader, NULL); - qstr qst = load_qstr(reader, qw); + size_t off = read_uint(reader); + qstr qst = load_qstr(reader); uint8_t *dest = fun_data + (off >> 2); if ((off & 3) == 0) { // Generic 16-bit link @@ -409,113 +296,92 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { } if (kind == MP_CODE_NATIVE_PY) { - // Extract prelude for later use - prelude_offset = read_uint(reader, NULL); + // Read prelude offset within fun_data, and extract scope flags. + prelude_offset = read_uint(reader); const byte *ip = fun_data + prelude_offset; - byte *ip_info = extract_prelude(&ip, &prelude); - // Load qstrs in prelude - load_prelude_qstrs(reader, qw, ip_info); + MP_BC_PRELUDE_SIG_DECODE(ip); + native_scope_flags = scope_flags; } else { - // Load basic scope info for viper and asm - prelude.scope_flags = read_uint(reader, NULL); - prelude.n_pos_args = 0; - prelude.n_kwonly_args = 0; + // Load basic scope info for viper and asm. + native_scope_flags = read_uint(reader); if (kind == MP_CODE_NATIVE_ASM) { - prelude.n_pos_args = read_uint(reader, NULL); - type_sig = read_uint(reader, NULL); + native_n_pos_args = read_uint(reader); + native_type_sig = read_uint(reader); } } #endif } - size_t n_obj = 0; - size_t n_raw_code = 0; - mp_uint_t *const_table = NULL; + size_t n_children = 0; + mp_raw_code_t **children = NULL; - if (kind != MP_CODE_NATIVE_ASM) { - // Load constant table for bytecode, native and viper - - // Number of entries in constant table - n_obj = read_uint(reader, NULL); - n_raw_code = read_uint(reader, NULL); - - // Allocate constant table - size_t n_alloc = prelude.n_pos_args + prelude.n_kwonly_args + n_obj + n_raw_code; - #if MICROPY_EMIT_MACHINE_CODE - if (kind != MP_CODE_BYTECODE) { - ++n_alloc; // additional entry for mp_fun_table - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { - ++n_alloc; // additional entry for rodata - } - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) { - ++n_alloc; // additional entry for BSS - } + #if MICROPY_EMIT_MACHINE_CODE + // Load optional BSS/rodata for viper. + uint8_t *rodata = NULL; + uint8_t *bss = NULL; + if (kind == MP_CODE_NATIVE_VIPER) { + size_t rodata_size = 0; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { + rodata_size = read_uint(reader); } - #endif - const_table = m_new(mp_uint_t, n_alloc); - mp_uint_t *ct = const_table; - - // Load function argument names (initial entries in const_table) - // (viper has n_pos_args=n_kwonly_args=0 so doesn't load any qstrs here) - for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { - *ct++ = (mp_uint_t)MP_OBJ_NEW_QSTR(load_qstr(reader, qw)); + size_t bss_size = 0; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERBSS) { + bss_size = read_uint(reader); } - #if MICROPY_EMIT_MACHINE_CODE - if (kind != MP_CODE_BYTECODE) { - // Populate mp_fun_table entry - *ct++ = (mp_uint_t)(uintptr_t)&mp_fun_table; - - // Allocate and load rodata if needed - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { - size_t size = read_uint(reader, NULL); - uint8_t *rodata = m_new(uint8_t, size); - read_bytes(reader, rodata, size); - *ct++ = (uintptr_t)rodata; + if (rodata_size + bss_size != 0) { + bss_size = (uintptr_t)MP_ALIGN(bss_size, sizeof(uintptr_t)); + uint8_t *data = m_new0(uint8_t, bss_size + rodata_size); + bss = data; + rodata = bss + bss_size; + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA) { + read_bytes(reader, rodata, rodata_size); } - // Allocate BSS if needed - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERBSS) { - size_t size = read_uint(reader, NULL); - uint8_t *bss = m_new0(uint8_t, size); - *ct++ = (uintptr_t)bss; - } + // Viper code with BSS/rodata should not have any children. + // Reuse the children pointer to reference the BSS/rodata + // memory so that it is not reclaimed by the GC. + assert(!has_children); + children = (void *)data; } - #endif + } + #endif - // Load constant objects and raw code children - for (size_t i = 0; i < n_obj; ++i) { - *ct++ = (mp_uint_t)load_obj(reader); - } - for (size_t i = 0; i < n_raw_code; ++i) { - *ct++ = (mp_uint_t)(uintptr_t)load_raw_code(reader, qw); + // Load children if any. + if (has_children) { + n_children = read_uint(reader); + children = m_new(mp_raw_code_t *, n_children); + for (size_t i = 0; i < n_children; ++i) { + children[i] = load_raw_code(reader); } } // Create raw_code and return it mp_raw_code_t *rc = mp_emit_glue_new_raw_code(); if (kind == MP_CODE_BYTECODE) { + const byte *ip = fun_data; + MP_BC_PRELUDE_SIG_DECODE(ip); // Assign bytecode to raw code object mp_emit_glue_assign_bytecode(rc, fun_data, #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS fun_data_len, #endif - const_table, + children, #if MICROPY_PERSISTENT_CODE_SAVE - n_obj, n_raw_code, + n_children, #endif - prelude.scope_flags); + scope_flags); #if MICROPY_EMIT_MACHINE_CODE } else { // Relocate and commit code to executable address space - reloc_info_t ri = {reader, const_table}; + reloc_info_t ri = {reader, rodata, bss}; #if defined(MP_PLAT_COMMIT_EXEC) - void *opt_ri = (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; + void *opt_ri = (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) ? &ri : NULL; fun_data = MP_PLAT_COMMIT_EXEC(fun_data, fun_data_len, opt_ri); #else - if (prelude.scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { + if (native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC) { #if MICROPY_PERSISTENT_CODE_TRACK_RELOC_CODE // If native code needs relocations then it's not guaranteed that a pointer to // the head of `buf` (containing the machine code) will be retained for the GC @@ -534,26 +400,27 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader, qstr_window_t *qw) { // Assign native code to raw code object mp_emit_glue_assign_native(rc, kind, - fun_data, fun_data_len, const_table, + fun_data, fun_data_len, + children, #if MICROPY_PERSISTENT_CODE_SAVE + n_children, prelude_offset, - n_obj, n_raw_code, - n_qstr_link, NULL, + 0, NULL, #endif - prelude.n_pos_args, prelude.scope_flags, type_sig); + native_scope_flags, native_n_pos_args, native_type_sig + ); #endif } return rc; } -mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *context) { byte header[4]; read_bytes(reader, header, sizeof(header)); if (header[0] != 'M' || header[1] != MPY_VERSION || MPY_FEATURE_DECODE_FLAGS(header[2]) != MPY_FEATURE_FLAGS - || header[3] > mp_small_int_bits() - || read_uint(reader, NULL) > QSTR_WINDOW_SIZE) { + || header[3] > mp_small_int_bits()) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy file")); } if (MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE) { @@ -562,25 +429,49 @@ mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader) { mp_raise_ValueError(MP_ERROR_TEXT("incompatible .mpy arch")); } } - qstr_window_t qw; - qw.idx = 0; - mp_raw_code_t *rc = load_raw_code(reader, &qw); + + size_t n_qstr = read_uint(reader); + size_t n_obj = read_uint(reader); + mp_module_context_alloc_tables(context, n_qstr, n_obj); + + // Load qstrs. + for (size_t i = 0; i < n_qstr; ++i) { + context->constants.qstr_table[i] = load_qstr(reader); + } + + // Load constant objects. + for (size_t i = 0; i < n_obj; ++i) { + context->constants.obj_table[i] = load_obj(reader); + } + + // Load top-level module. + mp_compiled_module_t cm2; + cm2.rc = load_raw_code(reader); + cm2.context = context; + + #if MICROPY_PERSISTENT_CODE_SAVE + cm2.has_native = MPY_FEATURE_DECODE_ARCH(header[2]) != MP_NATIVE_ARCH_NONE; + cm2.n_qstr = n_qstr; + cm2.n_obj = n_obj; + #endif + reader->close(reader->data); - return rc; + + return cm2; } -mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len) { +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *context) { mp_reader_t reader; mp_reader_new_mem(&reader, buf, len, 0); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, context); } #if MICROPY_HAS_FILE_READER -mp_raw_code_t *mp_raw_code_load_file(const char *filename) { +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *context) { mp_reader_t reader; mp_reader_new_file(&reader, filename); - return mp_raw_code_load(&reader); + return mp_raw_code_load(&reader, context); } #endif // MICROPY_HAS_FILE_READER @@ -607,26 +498,25 @@ STATIC void mp_print_uint(mp_print_t *print, size_t n) { print->print_strn(print->data, (char *)p, buf + sizeof(buf) - p); } -STATIC void save_qstr(mp_print_t *print, qstr_window_t *qw, qstr qst) { +STATIC void save_qstr(mp_print_t *print, qstr qst) { if (qst <= QSTR_LAST_STATIC) { // encode static qstr - byte buf[2] = {0, qst & 0xff}; - mp_print_bytes(print, buf, 2); - return; - } - size_t idx = qstr_window_insert(qw, qst); - if (idx < QSTR_WINDOW_SIZE) { - // qstr found in window, encode index to it - mp_print_uint(print, idx << 1 | 1); + mp_print_uint(print, qst << 1 | 1); return; } size_t len; const byte *str = qstr_data(qst, &len); mp_print_uint(print, len << 1); - mp_print_bytes(print, str, len); + mp_print_bytes(print, str, len + 1); // +1 to store null terminator } STATIC void save_obj(mp_print_t *print, mp_obj_t o) { + #if MICROPY_EMIT_MACHINE_CODE + if (o == MP_OBJ_FROM_PTR(&mp_fun_table)) { + byte obj_type = 't'; + mp_print_bytes(print, &obj_type, 1); + } else + #endif if (mp_obj_is_str_or_bytes(o)) { byte obj_type; if (mp_obj_is_str(o)) { @@ -638,7 +528,7 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) { const char *str = mp_obj_str_get_data(o, &len); mp_print_bytes(print, &obj_type, 1); mp_print_uint(print, len); - mp_print_bytes(print, (const byte *)str, len); + mp_print_bytes(print, (const byte *)str, len + 1); // +1 to store null terminator } else if (MP_OBJ_TO_PTR(o) == &mp_const_ellipsis_obj) { byte obj_type = 'e'; mp_print_bytes(print, &obj_type, 1); @@ -667,136 +557,45 @@ STATIC void save_obj(mp_print_t *print, mp_obj_t o) { } } -STATIC void save_prelude_qstrs(mp_print_t *print, qstr_window_t *qw, const byte *ip) { - save_qstr(print, qw, ip[0] | (ip[1] << 8)); // simple_name - save_qstr(print, qw, ip[2] | (ip[3] << 8)); // source_file -} - -STATIC void save_bytecode(mp_print_t *print, qstr_window_t *qw, const byte *ip, const byte *ip_top) { - while (ip < ip_top) { - size_t sz; - uint f = mp_opcode_format(ip, &sz, true); - if (f == MP_BC_FORMAT_QSTR) { - mp_print_bytes(print, ip, 1); - qstr qst = ip[1] | (ip[2] << 8); - save_qstr(print, qw, qst); - ip += 3; - sz -= 3; - } - mp_print_bytes(print, ip, sz); - ip += sz; - } -} - -STATIC void save_raw_code(mp_print_t *print, mp_raw_code_t *rc, qstr_window_t *qstr_window) { +STATIC void save_raw_code(mp_print_t *print, const mp_raw_code_t *rc) { // Save function kind and data length - mp_print_uint(print, (rc->fun_data_len << 2) | (rc->kind - MP_CODE_BYTECODE)); + mp_print_uint(print, (rc->fun_data_len << 3) | ((rc->n_children != 0) << 2) | (rc->kind - MP_CODE_BYTECODE)); - bytecode_prelude_t prelude; + // Save function code. + mp_print_bytes(print, rc->fun_data, rc->fun_data_len); - if (rc->kind == MP_CODE_BYTECODE) { - // Extract prelude - const byte *ip = rc->fun_data; - const byte *ip_info = extract_prelude(&ip, &prelude); - - // Save prelude - mp_print_bytes(print, rc->fun_data, ip_info - (const byte *)rc->fun_data); - save_prelude_qstrs(print, qstr_window, ip_info); - ip_info += 4; - mp_print_bytes(print, ip_info, ip - ip_info); - - // Save bytecode - const byte *ip_top = (const byte *)rc->fun_data + rc->fun_data_len; - save_bytecode(print, qstr_window, ip, ip_top); #if MICROPY_EMIT_MACHINE_CODE - } else { - // Save native code - mp_print_bytes(print, rc->fun_data, rc->fun_data_len); - - if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) { - // Save qstr link table for native code - mp_print_uint(print, rc->n_qstr); - for (size_t i = 0; i < rc->n_qstr; ++i) { - mp_print_uint(print, rc->qstr_link[i].off); - save_qstr(print, qstr_window, rc->qstr_link[i].qst); - } + if (rc->kind == MP_CODE_NATIVE_PY || rc->kind == MP_CODE_NATIVE_VIPER) { + // Save qstr link table for native code + mp_print_uint(print, rc->n_qstr); + for (size_t i = 0; i < rc->n_qstr; ++i) { + mp_print_uint(print, rc->qstr_link[i].off); + save_qstr(print, rc->qstr_link[i].qst); } - - if (rc->kind == MP_CODE_NATIVE_PY) { - // Save prelude size - mp_print_uint(print, rc->prelude_offset); - - // Extract prelude and save qstrs in prelude - const byte *ip = (const byte *)rc->fun_data + rc->prelude_offset; - const byte *ip_info = extract_prelude(&ip, &prelude); - save_prelude_qstrs(print, qstr_window, ip_info); - } else { - // Save basic scope info for viper and asm - mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG); - prelude.n_pos_args = 0; - prelude.n_kwonly_args = 0; - if (rc->kind == MP_CODE_NATIVE_ASM) { - mp_print_uint(print, rc->n_pos_args); - mp_print_uint(print, rc->type_sig); - } - } - #endif } - if (rc->kind != MP_CODE_NATIVE_ASM) { - // Save constant table for bytecode, native and viper - - // Number of entries in constant table - mp_print_uint(print, rc->n_obj); - mp_print_uint(print, rc->n_raw_code); - - const mp_uint_t *const_table = rc->const_table; - - // Save function argument names (initial entries in const_table) - // (viper has n_pos_args=n_kwonly_args=0 so doesn't save any qstrs here) - for (size_t i = 0; i < prelude.n_pos_args + prelude.n_kwonly_args; ++i) { - mp_obj_t o = (mp_obj_t)*const_table++; - save_qstr(print, qstr_window, MP_OBJ_QSTR_VALUE(o)); - } - - if (rc->kind != MP_CODE_BYTECODE) { - // Skip saving mp_fun_table entry - ++const_table; - } - - // Save constant objects and raw code children - for (size_t i = 0; i < rc->n_obj; ++i) { - save_obj(print, (mp_obj_t)*const_table++); + if (rc->kind == MP_CODE_NATIVE_PY) { + // Save prelude size + mp_print_uint(print, rc->prelude_offset); + } else if (rc->kind == MP_CODE_NATIVE_VIPER || rc->kind == MP_CODE_NATIVE_ASM) { + // Save basic scope info for viper and asm + mp_print_uint(print, rc->scope_flags & MP_SCOPE_FLAG_ALL_SIG); + if (rc->kind == MP_CODE_NATIVE_ASM) { + mp_print_uint(print, rc->n_pos_args); + mp_print_uint(print, rc->type_sig); } - for (size_t i = 0; i < rc->n_raw_code; ++i) { - save_raw_code(print, (mp_raw_code_t *)(uintptr_t)*const_table++, qstr_window); - } - } -} - -STATIC bool mp_raw_code_has_native(mp_raw_code_t *rc) { - if (rc->kind != MP_CODE_BYTECODE) { - return true; } + #endif - const byte *ip = rc->fun_data; - bytecode_prelude_t prelude; - extract_prelude(&ip, &prelude); - - const mp_uint_t *const_table = rc->const_table - + prelude.n_pos_args + prelude.n_kwonly_args - + rc->n_obj; - - for (size_t i = 0; i < rc->n_raw_code; ++i) { - if (mp_raw_code_has_native((mp_raw_code_t *)(uintptr_t)*const_table++)) { - return true; + if (rc->n_children) { + mp_print_uint(print, rc->n_children); + for (size_t i = 0; i < rc->n_children; ++i) { + save_raw_code(print, rc->children[i]); } } - - return false; } -void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { +void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print) { // header contains: // byte 'M' // byte version @@ -813,16 +612,27 @@ void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print) { mp_small_int_bits(), #endif }; - if (mp_raw_code_has_native(rc)) { + if (cm->has_native) { header[2] |= MPY_FEATURE_ENCODE_ARCH(MPY_FEATURE_ARCH_DYNAMIC); } mp_print_bytes(print, header, sizeof(header)); - mp_print_uint(print, QSTR_WINDOW_SIZE); - qstr_window_t qw; - qw.idx = 0; - memset(qw.window, 0, sizeof(qw.window)); - save_raw_code(print, rc, &qw); + // Number of entries in constant table. + mp_print_uint(print, cm->n_qstr); + mp_print_uint(print, cm->n_obj); + + // Save qstrs. + for (size_t i = 0; i < cm->n_qstr; ++i) { + save_qstr(print, cm->context->constants.qstr_table[i]); + } + + // Save constant objects. + for (size_t i = 0; i < cm->n_obj; ++i) { + save_obj(print, (mp_obj_t)cm->context->constants.obj_table[i]); + } + + // Save outer raw code, which will save all its child raw codes. + save_raw_code(print, cm->rc); } #if MICROPY_PERSISTENT_CODE_SAVE_FILE @@ -839,12 +649,12 @@ STATIC void fd_print_strn(void *env, const char *str, size_t len) { (void)ret; } -void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename) { +void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename) { MP_THREAD_GIL_EXIT(); int fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC, 0644); MP_THREAD_GIL_ENTER(); mp_print_t fd_print = {(void *)(intptr_t)fd, fd_print_strn}; - mp_raw_code_save(rc, &fd_print); + mp_raw_code_save(cm, &fd_print); MP_THREAD_GIL_EXIT(); close(fd); MP_THREAD_GIL_ENTER(); diff --git a/py/persistentcode.h b/py/persistentcode.h index 94fc3bf2d3..1991ba26ff 100644 --- a/py/persistentcode.h +++ b/py/persistentcode.h @@ -31,7 +31,7 @@ #include "py/emitglue.h" // The current version of .mpy files -#define MPY_VERSION 5 +#define MPY_VERSION 6 // Macros to encode/decode flags to/from the feature byte #define MPY_FEATURE_ENCODE_FLAGS(flags) (flags) @@ -102,12 +102,12 @@ enum { MP_NATIVE_ARCH_XTENSAWIN, }; -mp_raw_code_t *mp_raw_code_load(mp_reader_t *reader); -mp_raw_code_t *mp_raw_code_load_mem(const byte *buf, size_t len); -mp_raw_code_t *mp_raw_code_load_file(const char *filename); +mp_compiled_module_t mp_raw_code_load(mp_reader_t *reader, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_mem(const byte *buf, size_t len, mp_module_context_t *ctx); +mp_compiled_module_t mp_raw_code_load_file(const char *filename, mp_module_context_t *ctx); -void mp_raw_code_save(mp_raw_code_t *rc, mp_print_t *print); -void mp_raw_code_save_file(mp_raw_code_t *rc, const char *filename); +void mp_raw_code_save(mp_compiled_module_t *cm, mp_print_t *print); +void mp_raw_code_save_file(mp_compiled_module_t *cm, const char *filename); void mp_native_relocate(void *reloc, uint8_t *text, uintptr_t reloc_text); diff --git a/py/profile.c b/py/profile.c index d0ac99e074..4e23e9eac4 100644 --- a/py/profile.c +++ b/py/profile.c @@ -27,14 +27,16 @@ #include "py/profile.h" #include "py/bc0.h" #include "py/gc.h" +#include "py/objfun.h" #if MICROPY_PY_SYS_SETTRACE #define prof_trace_cb MP_STATE_THREAD(prof_trace_callback) +#define QSTR_MAP(context, idx) (context->constants.qstr_table[idx]) STATIC uint mp_prof_bytecode_lineno(const mp_raw_code_t *rc, size_t bc) { const mp_bytecode_prelude_t *prelude = &rc->prelude; - return mp_bytecode_get_source_line(prelude->line_info, bc); + return mp_bytecode_get_source_line(prelude->line_info, prelude->line_info_top, bc); } void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude) { @@ -50,13 +52,14 @@ void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelud MP_BC_PRELUDE_SIZE_DECODE(ip); - prelude->line_info = ip + 4; + prelude->line_info_top = ip + n_info; prelude->opcodes = ip + n_info + n_cell; - qstr block_name = ip[0] | (ip[1] << 8); - qstr source_file = ip[2] | (ip[3] << 8); - prelude->qstr_block_name = block_name; - prelude->qstr_source_file = source_file; + prelude->qstr_block_name_idx = mp_decode_uint_value(ip); + for (size_t i = 0; i < 1 + n_pos_args + n_kwonly_args; ++i) { + ip = mp_decode_uint_skip(ip); + } + prelude->line_info = ip; } /******************************************************************************/ @@ -69,22 +72,19 @@ STATIC void code_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t k const mp_bytecode_prelude_t *prelude = &rc->prelude; mp_printf(print, "<code object %q at 0x%p, file \"%q\", line %d>", - prelude->qstr_block_name, + QSTR_MAP(o->context, prelude->qstr_block_name_idx), o, - prelude->qstr_source_file, + QSTR_MAP(o->context, 0), rc->line_of_definition ); } -STATIC mp_obj_tuple_t *code_consts(const mp_raw_code_t *rc) { - const mp_bytecode_prelude_t *prelude = &rc->prelude; - int start = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj; - int stop = prelude->n_pos_args + prelude->n_kwonly_args + rc->n_obj + rc->n_raw_code; - mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(stop - start + 1, NULL)); +STATIC mp_obj_tuple_t *code_consts(const mp_module_context_t *context, const mp_raw_code_t *rc) { + mp_obj_tuple_t *consts = MP_OBJ_TO_PTR(mp_obj_new_tuple(rc->n_children + 1, NULL)); size_t const_no = 0; - for (int i = start; i < stop; ++i) { - mp_obj_t code = mp_obj_new_code((const mp_raw_code_t *)MP_OBJ_TO_PTR(rc->const_table[i])); + for (size_t i = 0; i < rc->n_children; ++i) { + mp_obj_t code = mp_obj_new_code(context, rc->children[i]); if (code == MP_OBJ_NULL) { m_malloc_fail(sizeof(mp_obj_code_t)); } @@ -149,16 +149,16 @@ STATIC void code_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { ); break; case MP_QSTR_co_consts: - dest[0] = MP_OBJ_FROM_PTR(code_consts(rc)); + dest[0] = MP_OBJ_FROM_PTR(code_consts(o->context, rc)); break; case MP_QSTR_co_filename: - dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_source_file); + dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, 0)); break; case MP_QSTR_co_firstlineno: dest[0] = MP_OBJ_NEW_SMALL_INT(mp_prof_bytecode_lineno(rc, 0)); break; case MP_QSTR_co_name: - dest[0] = MP_OBJ_NEW_QSTR(prelude->qstr_block_name); + dest[0] = MP_OBJ_NEW_QSTR(QSTR_MAP(o->context, prelude->qstr_block_name_idx)); break; case MP_QSTR_co_names: dest[0] = MP_OBJ_FROM_PTR(o->dict_locals); @@ -180,12 +180,13 @@ const mp_obj_type_t mp_type_settrace_codeobj = { .attr = code_attr, }; -mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc) { +mp_obj_t mp_obj_new_code(const mp_module_context_t *context, const mp_raw_code_t *rc) { mp_obj_code_t *o = m_new_obj_maybe(mp_obj_code_t); if (o == NULL) { return MP_OBJ_NULL; } o->base.type = &mp_type_settrace_codeobj; + o->context = context; o->rc = rc; o->dict_locals = mp_locals_get(); // this is a wrong! how to do this properly? o->lnotab = MP_OBJ_NULL; @@ -204,9 +205,9 @@ STATIC void frame_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t mp_printf(print, "<frame at 0x%p, file '%q', line %d, code %q>", frame, - prelude->qstr_source_file, + QSTR_MAP(code->context, 0), frame->lineno, - prelude->qstr_block_name + QSTR_MAP(code->context, prelude->qstr_block_name_idx) ); } @@ -229,7 +230,7 @@ STATIC void frame_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { dest[0] = MP_OBJ_FROM_PTR(o->code); break; case MP_QSTR_f_globals: - dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->globals); + dest[0] = MP_OBJ_FROM_PTR(o->code_state->fun_bc->context->module.globals); break; case MP_QSTR_f_lasti: dest[0] = MP_OBJ_NEW_SMALL_INT(o->lasti); @@ -258,7 +259,7 @@ mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state) { return MP_OBJ_NULL; } - mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->rc)); + mp_obj_code_t *code = o->code = MP_OBJ_TO_PTR(mp_obj_new_code(code_state->fun_bc->context, code_state->fun_bc->rc)); if (code == NULL) { return MP_OBJ_NULL; } diff --git a/py/profile.h b/py/profile.h index 64e207d04f..7f3f914034 100644 --- a/py/profile.h +++ b/py/profile.h @@ -34,7 +34,9 @@ #define mp_prof_is_executing MP_STATE_THREAD(prof_callback_is_executing) typedef struct _mp_obj_code_t { + // TODO this was 4 words mp_obj_base_t base; + const mp_module_context_t *context; const mp_raw_code_t *rc; mp_obj_dict_t *dict_locals; mp_obj_t lnotab; @@ -53,7 +55,7 @@ typedef struct _mp_obj_frame_t { void mp_prof_extract_prelude(const byte *bytecode, mp_bytecode_prelude_t *prelude); -mp_obj_t mp_obj_new_code(const mp_raw_code_t *rc); +mp_obj_t mp_obj_new_code(const mp_module_context_t *mc, const mp_raw_code_t *rc); mp_obj_t mp_obj_new_frame(const mp_code_state_t *code_state); // This is the implementation for the sys.settrace @@ -46,6 +46,7 @@ enum { }; typedef size_t qstr; +typedef uint16_t qstr_short_t; #if MICROPY_QSTR_BYTES_IN_HASH == 1 typedef uint8_t qstr_hash_t; diff --git a/py/scope.c b/py/scope.c index 98e02fb53f..8fc0943289 100644 --- a/py/scope.c +++ b/py/scope.c @@ -40,7 +40,7 @@ STATIC const uint8_t scope_simple_name_table[] = { [SCOPE_GEN_EXPR] = MP_QSTR__lt_genexpr_gt_, }; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options) { +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, mp_uint_t emit_options) { // Make sure those qstrs indeed fit in an uint8_t. MP_STATIC_ASSERT(MP_QSTR__lt_module_gt_ <= UINT8_MAX); MP_STATIC_ASSERT(MP_QSTR__lt_lambda_gt_ <= UINT8_MAX); @@ -52,7 +52,6 @@ scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_u scope_t *scope = m_new0(scope_t, 1); scope->kind = kind; scope->pn = pn; - scope->source_file = source_file; if (kind == SCOPE_FUNCTION || kind == SCOPE_CLASS) { assert(MP_PARSE_NODE_IS_STRUCT(pn)); scope->simple_name = MP_PARSE_NODE_LEAF_ARG(((mp_parse_node_struct_t *)pn)->nodes[0]); diff --git a/py/scope.h b/py/scope.h index edf164c4ad..5006deadea 100644 --- a/py/scope.h +++ b/py/scope.h @@ -75,7 +75,6 @@ typedef struct _scope_t { struct _scope_t *next; mp_parse_node_t pn; mp_raw_code_t *raw_code; - uint16_t source_file; // a qstr uint16_t simple_name; // a qstr uint16_t scope_flags; // see runtime0.h uint16_t emit_options; // see emitglue.h @@ -90,7 +89,7 @@ typedef struct _scope_t { id_info_t *id_info; } scope_t; -scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, qstr source_file, mp_uint_t emit_options); +scope_t *scope_new(scope_kind_t kind, mp_parse_node_t pn, mp_uint_t emit_options); void scope_free(scope_t *scope); id_info_t *scope_find_or_add_id(scope_t *scope, qstr qstr, id_info_kind_t kind); id_info_t *scope_find(scope_t *scope, qstr qstr); diff --git a/py/showbc.c b/py/showbc.c index da8077eccd..1300ac1fa3 100644 --- a/py/showbc.c +++ b/py/showbc.c @@ -41,43 +41,32 @@ #define DECODE_ULABEL do { unum = (ip[0] | (ip[1] << 8)); ip += 2; } while (0) #define DECODE_SLABEL do { unum = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2; } while (0) -#if MICROPY_PERSISTENT_CODE +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE #define DECODE_QSTR \ - qst = ip[0] | ip[1] << 8; \ - ip += 2; -#define DECODE_PTR \ - DECODE_UINT; \ - unum = mp_showbc_const_table[unum] -#define DECODE_OBJ \ DECODE_UINT; \ - unum = mp_showbc_const_table[unum] + qst = mp_showbc_constants->qstr_table[unum] #else -#define DECODE_QSTR { \ - qst = 0; \ - do { \ - qst = (qst << 7) + (*ip & 0x7f); \ - } while ((*ip++ & 0x80) != 0); \ -} -#define DECODE_PTR do { \ - ip = (byte *)MP_ALIGN(ip, sizeof(void *)); \ - unum = (uintptr_t)*(void **)ip; \ - ip += sizeof(void *); \ -} while (0) -#define DECODE_OBJ do { \ - ip = (byte *)MP_ALIGN(ip, sizeof(mp_obj_t)); \ - unum = (mp_uint_t)*(mp_obj_t *)ip; \ - ip += sizeof(mp_obj_t); \ -} while (0) +#define DECODE_QSTR \ + DECODE_UINT; \ + qst = unum; #endif -const byte *mp_showbc_code_start; -const mp_uint_t *mp_showbc_const_table; +#define DECODE_PTR \ + DECODE_UINT; + +#define DECODE_OBJ \ + DECODE_UINT; \ + unum = (mp_uint_t)mp_showbc_constants->obj_table[unum] -void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_uint_t *const_table) { +const byte * mp_showbc_code_start; +const mp_module_constants_t *mp_showbc_constants; + +#include "py/emitglue.h" +void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *ip, mp_uint_t len, const mp_module_constants_t *cm) { mp_showbc_code_start = ip; // Decode prelude @@ -85,16 +74,15 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i MP_BC_PRELUDE_SIZE_DECODE(ip); const byte *code_info = ip; - #if MICROPY_PERSISTENT_CODE - qstr block_name = code_info[0] | (code_info[1] << 8); - qstr source_file = code_info[2] | (code_info[3] << 8); - code_info += 4; - #else qstr block_name = mp_decode_uint(&code_info); - qstr source_file = mp_decode_uint(&code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + block_name = cm->qstr_table[block_name]; + qstr source_file = cm->qstr_table[0]; + #else + qstr source_file = cm->source_file; #endif - mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p " UINT_FMT " bytes)\n", - qstr_str(source_file), qstr_str(block_name), descr, mp_showbc_code_start, len); + mp_printf(print, "File %s, code block '%s' (descriptor: %p, bytecode @%p " UINT_FMT " bytes)\n",// rct=%p\n", + qstr_str(source_file), qstr_str(block_name), descr, mp_showbc_code_start, len);// , ((mp_raw_code_t*)descr)->children); // raw bytecode dump size_t prelude_size = ip - mp_showbc_code_start + n_info + n_cell; @@ -111,7 +99,11 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i // bytecode prelude: arg names (as qstr objects) mp_printf(print, "arg names:"); for (mp_uint_t i = 0; i < n_pos_args + n_kwonly_args; i++) { - mp_printf(print, " %s", qstr_str(MP_OBJ_QSTR_VALUE(const_table[i]))); + qstr qst = mp_decode_uint(&code_info); + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + qst = cm->qstr_table[qst]; + #endif + mp_printf(print, " %s", qstr_str(qst)); } mp_printf(print, "\n"); @@ -120,6 +112,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i // skip over code_info ip += n_info; + const byte *line_info_top = ip; // bytecode prelude: initialise closed over variables for (size_t i = 0; i < n_cell; ++i) { @@ -132,7 +125,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i mp_int_t bc = 0; mp_uint_t source_line = 1; mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); - for (const byte *ci = code_info; *ci;) { + for (const byte *ci = code_info; ci < line_info_top;) { if ((ci[0] & 0x80) == 0) { // 0b0LLBBBBB encoding bc += ci[0] & 0x1f; @@ -147,7 +140,7 @@ void mp_bytecode_print(const mp_print_t *print, const void *descr, const byte *i mp_printf(print, " bc=" INT_FMT " line=" UINT_FMT "\n", bc, source_line); } } - mp_bytecode_print2(print, ip, len - prelude_size, const_table); + mp_bytecode_print2(print, ip, len - prelude_size, cm); } const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { @@ -535,9 +528,9 @@ const byte *mp_bytecode_print_str(const mp_print_t *print, const byte *ip) { return ip; } -void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_uint_t *const_table) { +void mp_bytecode_print2(const mp_print_t *print, const byte *ip, size_t len, const mp_module_constants_t *cm) { mp_showbc_code_start = ip; - mp_showbc_const_table = const_table; + mp_showbc_constants = cm; while (ip < len + mp_showbc_code_start) { mp_printf(print, "%02u ", (uint)(ip - mp_showbc_code_start)); ip = mp_bytecode_print_str(print, ip); @@ -31,9 +31,9 @@ #include "py/emitglue.h" #include "py/objtype.h" +#include "py/objfun.h" #include "py/runtime.h" #include "py/bc0.h" -#include "py/bc.h" #include "py/profile.h" // *FORMAT-OFF* @@ -44,7 +44,7 @@ #else #define TRACE_PREFIX mp_printf(&mp_plat_print, "sp=%d ", (int)(sp - &code_state->state[0] + 1)) #endif -#define TRACE(ip) TRACE_PREFIX; mp_bytecode_print2(&mp_plat_print, ip, 1, code_state->fun_bc->const_table); +#define TRACE(ip) TRACE_PREFIX; mp_bytecode_print2(&mp_plat_print, ip, 1, &code_state->fun_bc->context->constants); #else #define TRACE(ip) #endif @@ -64,35 +64,28 @@ #define DECODE_ULABEL size_t ulab = (ip[0] | (ip[1] << 8)); ip += 2 #define DECODE_SLABEL size_t slab = (ip[0] | (ip[1] << 8)) - 0x8000; ip += 2 -#if MICROPY_PERSISTENT_CODE +#if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE #define DECODE_QSTR \ - qstr qst = ip[0] | ip[1] << 8; \ - ip += 2; -#define DECODE_PTR \ - DECODE_UINT; \ - void *ptr = (void*)(uintptr_t)code_state->fun_bc->const_table[unum] -#define DECODE_OBJ \ DECODE_UINT; \ - mp_obj_t obj = (mp_obj_t)code_state->fun_bc->const_table[unum] + qstr qst = qstr_table[unum] #else -#define DECODE_QSTR qstr qst = 0; \ - do { \ - qst = (qst << 7) + (*ip & 0x7f); \ - } while ((*ip++ & 0x80) != 0) -#define DECODE_PTR \ - ip = (byte*)MP_ALIGN(ip, sizeof(void*)); \ - void *ptr = *(void**)ip; \ - ip += sizeof(void*) -#define DECODE_OBJ \ - ip = (byte*)MP_ALIGN(ip, sizeof(mp_obj_t)); \ - mp_obj_t obj = *(mp_obj_t*)ip; \ - ip += sizeof(mp_obj_t) +#define DECODE_QSTR \ + DECODE_UINT; \ + qstr qst = unum; #endif +#define DECODE_PTR \ + DECODE_UINT; \ + void *ptr = (void *)(uintptr_t)code_state->fun_bc->child_table[unum] + +#define DECODE_OBJ \ + DECODE_UINT; \ + mp_obj_t obj = (mp_obj_t)code_state->fun_bc->context->constants.obj_table[unum] + #define PUSH(val) *++sp = (val) #define POP() (*sp--) #define TOP() (*sp) @@ -255,6 +248,9 @@ outer_dispatch_loop: // local variables that are not visible to the exception handler const byte *ip = code_state->ip; mp_obj_t *sp = code_state->sp; + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + const qstr_short_t *qstr_table = code_state->fun_bc->context->constants.qstr_table; + #endif mp_obj_t obj_shared; MICROPY_VM_HOOK_INIT @@ -859,15 +855,15 @@ unwind_jump:; ENTRY(MP_BC_MAKE_FUNCTION): { DECODE_PTR; - PUSH(mp_make_function_from_raw_code(ptr, MP_OBJ_NULL, MP_OBJ_NULL)); + PUSH(mp_make_function_from_raw_code(ptr, code_state->fun_bc->context, NULL)); DISPATCH(); } ENTRY(MP_BC_MAKE_FUNCTION_DEFARGS): { DECODE_PTR; // Stack layout: def_tuple def_dict <- TOS - mp_obj_t def_dict = POP(); - SET_TOP(mp_make_function_from_raw_code(ptr, TOP(), def_dict)); + sp -= 1; + SET_TOP(mp_make_function_from_raw_code(ptr, code_state->fun_bc->context, sp)); DISPATCH(); } @@ -876,7 +872,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: closed_overs <- TOS sp -= n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->context, n_closed_over, sp)); DISPATCH(); } @@ -885,7 +881,7 @@ unwind_jump:; size_t n_closed_over = *ip++; // Stack layout: def_tuple def_dict closed_overs <- TOS sp -= 2 + n_closed_over - 1; - SET_TOP(mp_make_closure_from_raw_code(ptr, 0x100 | n_closed_over, sp)); + SET_TOP(mp_make_closure_from_raw_code(ptr, code_state->fun_bc->context, 0x100 | n_closed_over, sp)); DISPATCH(); } @@ -1384,23 +1380,20 @@ unwind_loop: const byte *ip = code_state->fun_bc->bytecode; MP_BC_PRELUDE_SIG_DECODE(ip); MP_BC_PRELUDE_SIZE_DECODE(ip); + const byte *line_info_top = ip + n_info; const byte *bytecode_start = ip + n_info + n_cell; - #if !MICROPY_PERSISTENT_CODE - // so bytecode is aligned - bytecode_start = MP_ALIGN(bytecode_start, sizeof(mp_uint_t)); - #endif size_t bc = code_state->ip - bytecode_start; - #if MICROPY_PERSISTENT_CODE - qstr block_name = ip[0] | (ip[1] << 8); - qstr source_file = ip[2] | (ip[3] << 8); - ip += 4; - #else qstr block_name = mp_decode_uint_value(ip); - ip = mp_decode_uint_skip(ip); - qstr source_file = mp_decode_uint_value(ip); - ip = mp_decode_uint_skip(ip); + for (size_t i = 0; i < 1 + n_pos_args + n_kwonly_args; ++i) { + ip = mp_decode_uint_skip(ip); + } + #if MICROPY_EMIT_BYTECODE_USES_QSTR_TABLE + block_name = code_state->fun_bc->context->constants.qstr_table[block_name]; + qstr source_file = code_state->fun_bc->context->constants.qstr_table[0]; + #else + qstr source_file = code_state->fun_bc->context->constants.source_file; #endif - size_t source_line = mp_bytecode_get_source_line(ip, bc); + size_t source_line = mp_bytecode_get_source_line(ip, line_info_top, bc); mp_obj_exception_add_traceback(MP_OBJ_FROM_PTR(nlr.ret_val), source_file, source_line, block_name); } diff --git a/shared/runtime/pyexec.c b/shared/runtime/pyexec.c index 5f05c1da3e..ae6dd770b1 100644 --- a/shared/runtime/pyexec.c +++ b/shared/runtime/pyexec.c @@ -83,7 +83,11 @@ STATIC int parse_compile_execute(const void *source, mp_parse_input_kind_t input #if MICROPY_MODULE_FROZEN_MPY if (exec_flags & EXEC_FLAG_SOURCE_IS_RAW_CODE) { // source is a raw_code object, create the function - module_fun = mp_make_function_from_raw_code(source, MP_OBJ_NULL, MP_OBJ_NULL); + const mp_frozen_module_t *frozen = source; + mp_module_context_t *ctx = m_new_obj(mp_module_context_t); + ctx->module.globals = mp_globals_get(); + ctx->constants = frozen->constants; + module_fun = mp_make_function_from_raw_code(frozen->rc, ctx, NULL); } else #endif { diff --git a/tests/cmdline/cmd_parsetree.py.exp b/tests/cmdline/cmd_parsetree.py.exp index bee4fc99d1..ac16f6d88d 100644 --- a/tests/cmdline/cmd_parsetree.py.exp +++ b/tests/cmdline/cmd_parsetree.py.exp @@ -39,50 +39,52 @@ [ 13] \(rule\|arglist\)(164) (n=1) id(b) ---------------- -File cmdline/cmd_parsetree.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_parsetree.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ 64 bytes) +Raw bytecode (code_info_size=13, bytecode_size=51): + 20 16 01 60 28 23 23 24 24 24 24 24 25 2a 00 5f + 4b 05 00 16 02 42 f8 7f 51 16 03 10 04 16 05 23 + 00 16 06 23 01 16 07 23 02 16 08 23 03 16 09 22 + 80 7b 16 0a 23 04 14 0b 11 05 36 01 16 0c 51 63 arg names: (N_STATE 5) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=4 - bc=9 line=5 - bc=12 line=6 - bc=16 line=7 - bc=22 line=8 - bc=27 line=9 - bc=32 line=10 - bc=37 line=11 - bc=42 line=12 - bc=48 line=13 + bc=8 line=5 + bc=11 line=6 + bc=14 line=7 + bc=18 line=8 + bc=22 line=9 + bc=26 line=10 + bc=30 line=11 + bc=34 line=12 + bc=39 line=13 00 BUILD_TUPLE 0 02 GET_ITER_STACK -03 FOR_ITER 12 +03 FOR_ITER 11 06 STORE_NAME i -09 JUMP 3 -12 LOAD_CONST_NONE -13 STORE_NAME a -16 LOAD_CONST_STRING 'str' -19 STORE_NAME b -22 LOAD_CONST_OBJ \.\+ -24 STORE_NAME c -27 LOAD_CONST_OBJ \.\+ -29 STORE_NAME d -32 LOAD_CONST_OBJ \.\+ -34 STORE_NAME e -37 LOAD_CONST_OBJ \.\+ -39 STORE_NAME f -42 LOAD_CONST_SMALL_INT 123 -45 STORE_NAME g -48 LOAD_CONST_OBJ \.\+ -50 LOAD_METHOD format -53 LOAD_NAME b -56 CALL_METHOD n=1 nkw=0 -58 STORE_NAME h -61 LOAD_CONST_NONE -62 RETURN_VALUE +08 JUMP 3 +11 LOAD_CONST_NONE +12 STORE_NAME a +14 LOAD_CONST_STRING 'str' +16 STORE_NAME b +18 LOAD_CONST_OBJ \.\+='a very long str that will not be interned' +20 STORE_NAME c +22 LOAD_CONST_OBJ \.\+=b'bytes' +24 STORE_NAME d +26 LOAD_CONST_OBJ \.\+=b'a very long bytes that will not be interned' +28 STORE_NAME e +30 LOAD_CONST_OBJ \.\+=123456789012345678901234567890 +32 STORE_NAME f +34 LOAD_CONST_SMALL_INT 123 +37 STORE_NAME g +39 LOAD_CONST_OBJ \.\+="fstring: '{}'" +41 LOAD_METHOD format +43 LOAD_NAME b +45 CALL_METHOD n=1 nkw=0 +47 STORE_NAME h +49 LOAD_CONST_NONE +50 RETURN_VALUE mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ GC: total: \\d\+, used: \\d\+, free: \\d\+ diff --git a/tests/cmdline/cmd_showbc.py.exp b/tests/cmdline/cmd_showbc.py.exp index 22712b79ee..031820fcd9 100644 --- a/tests/cmdline/cmd_showbc.py.exp +++ b/tests/cmdline/cmd_showbc.py.exp @@ -1,51 +1,144 @@ -File cmdline/cmd_showbc.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ 63 bytes) +Raw bytecode (code_info_size=18, bytecode_size=45): + 10 20 01 60 20 84 7d 64 60 88 07 64 60 69 20 62 + 64 20 32 00 16 02 32 01 16 02 81 2a 01 53 33 02 + 16 02 32 03 16 02 54 32 04 10 03 34 02 16 03 19 + 03 32 05 16 02 80 10 04 2a 01 1b 05 69 51 63 arg names: (N_STATE 3) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=160 + bc=0 line=4 + bc=0 line=5 + bc=4 line=130 + bc=8 line=133 + bc=8 line=136 + bc=16 line=143 + bc=20 line=146 + bc=20 line=149 + bc=29 line=152 + bc=29 line=153 + bc=31 line=156 + bc=35 line=159 + bc=35 line=160 00 MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ BUILD_TUPLE 1 -\\d\+ LOAD_NULL -\\d\+ MAKE_FUNCTION_DEFARGS \.\+ -\\d\+ STORE_NAME f -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_BUILD_CLASS -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ LOAD_CONST_STRING 'Class' -\\d\+ CALL_FUNCTION n=2 nkw=0 -\\d\+ STORE_NAME Class -\\d\+ DELETE_NAME Class -\\d\+ MAKE_FUNCTION \.\+ -\\d\+ STORE_NAME f -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_STRING '*' -\\d\+ BUILD_TUPLE 1 -\\d\+ IMPORT_NAME 'sys' -\\d\+ IMPORT_STAR -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): +02 STORE_NAME f +04 MAKE_FUNCTION \.\+ +06 STORE_NAME f +08 LOAD_CONST_SMALL_INT 1 +09 BUILD_TUPLE 1 +11 LOAD_NULL +12 MAKE_FUNCTION_DEFARGS \.\+ +14 STORE_NAME f +16 MAKE_FUNCTION \.\+ +18 STORE_NAME f +20 LOAD_BUILD_CLASS +21 MAKE_FUNCTION \.\+ +23 LOAD_CONST_STRING 'Class' +25 CALL_FUNCTION n=2 nkw=0 +27 STORE_NAME Class +29 DELETE_NAME Class +31 MAKE_FUNCTION \.\+ +33 STORE_NAME f +35 LOAD_CONST_SMALL_INT 0 +36 LOAD_CONST_STRING '*' +38 BUILD_TUPLE 1 +40 IMPORT_NAME 'sys' +42 IMPORT_STAR +43 LOAD_CONST_NONE +44 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 48\[24\] bytes) +Raw bytecode (code_info_size=8\[46\], bytecode_size=398): + a8 12 9\[bf\] 03 02 60 60 26 22 24 64 22 26 25 25 24 + 26 23 63 22 22 25 23 23 31 6d 25 65 25 25 69 68 + 26 65 27 6a 62 20 23 62 2a 29 69 24 25 28 67 26 ######## -\.\+rg names: +\.\+81 63 +arg names: (N_STATE 22) (N_EXC_STACK 2) (INIT_CELL 14) (INIT_CELL 15) (INIT_CELL 16) bc=0 line=1 + bc=0 line=4 + bc=0 line=7 + bc=6 line=8 + bc=8 line=9 + bc=12 line=10 + bc=16 line=13 + bc=18 line=14 + bc=24 line=15 + bc=29 line=16 + bc=34 line=17 + bc=38 line=18 + bc=44 line=19 + bc=47 line=20 + bc=50 line=23 + bc=52 line=24 + bc=54 line=25 + bc=59 line=26 + bc=62 line=27 + bc=65 line=28 + bc=82 line=29 + bc=95 line=32 + bc=100 line=33 + bc=105 line=36 + bc=110 line=37 + bc=115 line=38 + bc=124 line=41 + bc=132 line=44 + bc=138 line=45 + bc=143 line=48 + bc=150 line=49 + bc=160 line=52 + bc=162 line=55 + bc=162 line=56 + bc=165 line=57 + bc=167 line=60 + bc=177 line=61 + bc=186 line=62 + bc=195 line=65 + bc=199 line=66 + bc=204 line=67 + bc=212 line=68 + bc=219 line=71 + bc=225 line=72 + bc=232 line=73 + bc=242 line=74 + bc=250 line=77 + bc=254 line=78 + bc=260 line=80 + bc=263 line=81 + bc=266 line=82 + bc=273 line=83 + bc=276 line=84 + bc=283 line=85 + bc=289 line=88 + bc=296 line=89 + bc=301 line=92 + bc=307 line=93 + bc=310 line=94 +######## + bc=321 line=96 + bc=329 line=98 + bc=332 line=99 + bc=335 line=100 + bc=338 line=101 ######## - bc=\\d\+ line=127 + bc=346 line=103 + bc=354 line=106 + bc=359 line=107 + bc=365 line=110 + bc=368 line=111 + bc=374 line=114 + bc=374 line=117 + bc=379 line=118 + bc=391 line=121 + bc=391 line=122 + bc=392 line=123 + bc=394 line=126 + bc=396 line=127 00 LOAD_CONST_NONE 01 LOAD_CONST_FALSE 02 BINARY_OP 27 __add__ @@ -80,247 +173,251 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 42 STORE_MAP 43 STORE_FAST 3 44 LOAD_CONST_STRING 'a' -47 STORE_FAST 4 -48 LOAD_CONST_OBJ \.\+ -\\d\+ STORE_FAST 5 -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ STORE_FAST 6 -\\d\+ LOAD_CONST_SMALL_INT 2 -\\d\+ STORE_FAST 7 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -58 BINARY_OP 27 __add__ -\\d\+ STORE_FAST 8 -\\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 1 __neg__ -\\d\+ STORE_FAST 9 -\\d\+ LOAD_FAST 0 -\\d\+ UNARY_OP 3 -\\d\+ STORE_FAST 10 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ DUP_TOP -\\d\+ ROT_THREE -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP_IF_FALSE_OR_POP \\d\+ -\\d\+ LOAD_FAST 1 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP \\d\+ -\\d\+ ROT_TWO -\\d\+ POP_TOP -\\d\+ STORE_FAST 10 -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ JUMP_IF_FALSE_OR_POP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 1 -\\d\+ BINARY_OP 2 __eq__ -\\d\+ UNARY_OP 3 -\\d\+ STORE_FAST 10 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_ATTR c -\\d\+ STORE_FAST 11 -\\d\+ LOAD_FAST 11 -\\d\+ LOAD_DEREF 14 -\\d\+ STORE_ATTR c -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_SUBSCR -\\d\+ STORE_FAST 12 -\\d\+ LOAD_FAST 12 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ STORE_SUBSCR -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ DUP_TOP_TWO -\\d\+ LOAD_SUBSCR -\\d\+ LOAD_FAST 12 -\\d\+ BINARY_OP 14 __iadd__ -\\d\+ ROT_THREE -\\d\+ STORE_SUBSCR -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_CONST_NONE -\\d\+ LOAD_CONST_NONE -\\d\+ BUILD_SLICE 2 -\\d\+ LOAD_SUBSCR -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 1 -\\d\+ UNPACK_SEQUENCE 2 -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ UNPACK_EX 1 -\\d\+ STORE_FAST 0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ ROT_TWO -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ LOAD_FAST 1 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_FAST 0 -\\d\+ ROT_THREE -\\d\+ ROT_TWO -\\d\+ STORE_FAST 0 -\\d\+ STORE_DEREF 14 -\\d\+ STORE_FAST 1 -\\d\+ DELETE_FAST 0 -\\d\+ LOAD_FAST 0 -\\d\+ STORE_GLOBAL gl -\\d\+ DELETE_GLOBAL gl -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ GET_ITER -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 14 -\\d\+ LOAD_FAST 15 -\\d\+ MAKE_CLOSURE \.\+ 2 -\\d\+ LOAD_FAST 2 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 0 -\\d\+ CALL_FUNCTION n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_FUNCTION n=1 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_CONST_STRING 'b' -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_FUNCTION n=0 nkw=1 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ LOAD_NULL -\\d\+ CALL_FUNCTION_VAR_KW n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ CALL_METHOD n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_METHOD n=1 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_CONST_STRING 'c' -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ CALL_METHOD n=0 nkw=1 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_METHOD b -\\d\+ LOAD_FAST 1 -\\d\+ LOAD_NULL -\\d\+ CALL_METHOD_VAR_KW n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_FALSE \\d\+ -\\d\+ LOAD_DEREF 16 -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ LOAD_GLOBAL y -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_FALSE \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ JUMP_IF_TRUE_OR_POP \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ STORE_FAST 0 -\\d\+ LOAD_DEREF 14 -\\d\+ GET_ITER_STACK -\\d\+ FOR_ITER \\d\+ -\\d\+ STORE_FAST 0 -\\d\+ LOAD_FAST 1 -\\d\+ POP_TOP -\\d\+ JUMP \\d\+ -\\d\+ SETUP_FINALLY \\d\+ -\\d\+ SETUP_EXCEPT \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ JUMP \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ END_FINALLY -\\d\+ LOAD_CONST_NONE -\\d\+ LOAD_FAST 1 -\\d\+ POP_TOP -\\d\+ END_FINALLY -\\d\+ JUMP \\d\+ -\\d\+ SETUP_EXCEPT \\d\+ -\\d\+ UNWIND_JUMP \\d\+ 1 -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ POP_TOP -\\d\+ POP_EXCEPT_JUMP \\d\+ -\\d\+ END_FINALLY -\\d\+ LOAD_FAST 0 -\\d\+ POP_JUMP_IF_TRUE \\d\+ -\\d\+ LOAD_FAST 0 -\\d\+ SETUP_WITH \\d\+ -\\d\+ POP_TOP -\\d\+ LOAD_DEREF 14 -\\d\+ POP_TOP -\\d\+ LOAD_CONST_NONE -\\d\+ WITH_CLEANUP -\\d\+ END_FINALLY -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ STORE_DEREF 16 -\\d\+ LOAD_FAST_N 16 -\\d\+ MAKE_CLOSURE \.\+ 1 -\\d\+ STORE_FAST 13 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_NONE -\\d\+ IMPORT_NAME 'a' -\\d\+ STORE_FAST 0 -\\d\+ LOAD_CONST_SMALL_INT 0 -\\d\+ LOAD_CONST_STRING 'b' -\\d\+ BUILD_TUPLE 1 -\\d\+ IMPORT_NAME 'a' -\\d\+ IMPORT_FROM 'b' -\\d\+ STORE_DEREF 14 -\\d\+ POP_TOP -\\d\+ RAISE_LAST -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ RAISE_OBJ -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -\\d\+ LOAD_CONST_SMALL_INT 1 -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+rg names: +46 STORE_FAST 4 +47 LOAD_CONST_OBJ \.\+=b'a' +49 STORE_FAST 5 +50 LOAD_CONST_SMALL_INT 1 +51 STORE_FAST 6 +52 LOAD_CONST_SMALL_INT 2 +53 STORE_FAST 7 +54 LOAD_FAST 0 +55 LOAD_DEREF 14 +57 BINARY_OP 27 __add__ +58 STORE_FAST 8 +59 LOAD_FAST 0 +60 UNARY_OP 1 __neg__ +61 STORE_FAST 9 +62 LOAD_FAST 0 +63 UNARY_OP 3 +64 STORE_FAST 10 +65 LOAD_FAST 0 +66 LOAD_DEREF 14 +68 DUP_TOP +69 ROT_THREE +70 BINARY_OP 2 __eq__ +71 JUMP_IF_FALSE_OR_POP 79 +74 LOAD_FAST 1 +75 BINARY_OP 2 __eq__ +76 JUMP 81 +79 ROT_TWO +80 POP_TOP +81 STORE_FAST 10 +82 LOAD_FAST 0 +83 LOAD_DEREF 14 +85 BINARY_OP 2 __eq__ +86 JUMP_IF_FALSE_OR_POP 93 +89 LOAD_DEREF 14 +91 LOAD_FAST 1 +92 BINARY_OP 2 __eq__ +93 UNARY_OP 3 +94 STORE_FAST 10 +95 LOAD_DEREF 14 +97 LOAD_ATTR c +99 STORE_FAST 11 +100 LOAD_FAST 11 +101 LOAD_DEREF 14 +103 STORE_ATTR c +105 LOAD_DEREF 14 +107 LOAD_CONST_SMALL_INT 0 +108 LOAD_SUBSCR +109 STORE_FAST 12 +110 LOAD_FAST 12 +111 LOAD_DEREF 14 +113 LOAD_CONST_SMALL_INT 0 +114 STORE_SUBSCR +115 LOAD_DEREF 14 +117 LOAD_CONST_SMALL_INT 0 +118 DUP_TOP_TWO +119 LOAD_SUBSCR +120 LOAD_FAST 12 +121 BINARY_OP 14 __iadd__ +122 ROT_THREE +123 STORE_SUBSCR +124 LOAD_DEREF 14 +126 LOAD_CONST_NONE +127 LOAD_CONST_NONE +128 BUILD_SLICE 2 +130 LOAD_SUBSCR +131 STORE_FAST 0 +132 LOAD_FAST 1 +133 UNPACK_SEQUENCE 2 +135 STORE_FAST 0 +136 STORE_DEREF 14 +138 LOAD_FAST 0 +139 UNPACK_EX 1 +141 STORE_FAST 0 +142 STORE_FAST 0 +143 LOAD_DEREF 14 +145 LOAD_FAST 0 +146 ROT_TWO +147 STORE_FAST 0 +148 STORE_DEREF 14 +150 LOAD_FAST 1 +151 LOAD_DEREF 14 +153 LOAD_FAST 0 +154 ROT_THREE +155 ROT_TWO +156 STORE_FAST 0 +157 STORE_DEREF 14 +159 STORE_FAST 1 +160 DELETE_FAST 0 +162 LOAD_FAST 0 +163 STORE_GLOBAL gl +165 DELETE_GLOBAL gl +167 LOAD_FAST 14 +168 LOAD_FAST 15 +169 MAKE_CLOSURE \.\+ 2 +172 LOAD_FAST 2 +173 GET_ITER +174 CALL_FUNCTION n=1 nkw=0 +176 STORE_FAST 0 +177 LOAD_FAST 14 +178 LOAD_FAST 15 +179 MAKE_CLOSURE \.\+ 2 +182 LOAD_FAST 2 +183 CALL_FUNCTION n=1 nkw=0 +185 STORE_FAST 0 +186 LOAD_FAST 14 +187 LOAD_FAST 15 +188 MAKE_CLOSURE \.\+ 2 +191 LOAD_FAST 2 +192 CALL_FUNCTION n=1 nkw=0 +194 STORE_FAST 0 +195 LOAD_FAST 0 +196 CALL_FUNCTION n=0 nkw=0 +198 POP_TOP +199 LOAD_FAST 0 +200 LOAD_CONST_SMALL_INT 1 +201 CALL_FUNCTION n=1 nkw=0 +203 POP_TOP +204 LOAD_FAST 0 +205 LOAD_CONST_STRING 'b' +207 LOAD_CONST_SMALL_INT 1 +208 CALL_FUNCTION n=0 nkw=1 +211 POP_TOP +212 LOAD_FAST 0 +213 LOAD_DEREF 14 +215 LOAD_NULL +216 CALL_FUNCTION_VAR_KW n=0 nkw=0 +218 POP_TOP +219 LOAD_FAST 0 +220 LOAD_METHOD b +222 CALL_METHOD n=0 nkw=0 +224 POP_TOP +225 LOAD_FAST 0 +226 LOAD_METHOD b +228 LOAD_CONST_SMALL_INT 1 +229 CALL_METHOD n=1 nkw=0 +231 POP_TOP +232 LOAD_FAST 0 +233 LOAD_METHOD b +235 LOAD_CONST_STRING 'c' +237 LOAD_CONST_SMALL_INT 1 +238 CALL_METHOD n=0 nkw=1 +241 POP_TOP +242 LOAD_FAST 0 +243 LOAD_METHOD b +245 LOAD_FAST 1 +246 LOAD_NULL +247 CALL_METHOD_VAR_KW n=0 nkw=0 +249 POP_TOP +250 LOAD_FAST 0 +251 POP_JUMP_IF_FALSE 260 +254 LOAD_DEREF 16 +256 POP_TOP +257 JUMP 263 +260 LOAD_GLOBAL y +262 POP_TOP +263 JUMP 269 +266 LOAD_DEREF 14 +268 POP_TOP +269 LOAD_FAST 0 +270 POP_JUMP_IF_TRUE 266 +273 JUMP 279 +276 LOAD_DEREF 14 +278 POP_TOP +279 LOAD_FAST 0 +280 POP_JUMP_IF_FALSE 276 +283 LOAD_FAST 0 +284 JUMP_IF_TRUE_OR_POP 288 +287 LOAD_FAST 0 +288 STORE_FAST 0 +289 LOAD_DEREF 14 +291 GET_ITER_STACK +292 FOR_ITER 301 +295 STORE_FAST 0 +296 LOAD_FAST 1 +297 POP_TOP +298 JUMP 292 +301 SETUP_FINALLY 329 +304 SETUP_EXCEPT 320 +307 JUMP 313 +310 JUMP 317 +313 LOAD_FAST 0 +314 POP_JUMP_IF_TRUE 310 +317 POP_EXCEPT_JUMP 328 +320 POP_TOP +321 LOAD_DEREF 14 +323 POP_TOP +324 POP_EXCEPT_JUMP 328 +327 END_FINALLY +328 LOAD_CONST_NONE +329 LOAD_FAST 1 +330 POP_TOP +331 END_FINALLY +332 JUMP 350 +335 SETUP_EXCEPT 345 +338 UNWIND_JUMP 354 1 +342 POP_EXCEPT_JUMP 350 +345 POP_TOP +346 POP_EXCEPT_JUMP 350 +349 END_FINALLY +350 LOAD_FAST 0 +351 POP_JUMP_IF_TRUE 335 +354 LOAD_FAST 0 +355 SETUP_WITH 363 +358 POP_TOP +359 LOAD_DEREF 14 +361 POP_TOP +362 LOAD_CONST_NONE +363 WITH_CLEANUP +364 END_FINALLY +365 LOAD_CONST_SMALL_INT 1 +366 STORE_DEREF 16 +368 LOAD_FAST_N 16 +370 MAKE_CLOSURE \.\+ 1 +373 STORE_FAST 13 +374 LOAD_CONST_SMALL_INT 0 +375 LOAD_CONST_NONE +376 IMPORT_NAME 'a' +378 STORE_FAST 0 +379 LOAD_CONST_SMALL_INT 0 +380 LOAD_CONST_STRING 'b' +382 BUILD_TUPLE 1 +384 IMPORT_NAME 'a' +386 IMPORT_FROM 'b' +388 STORE_DEREF 14 +390 POP_TOP +391 RAISE_LAST +392 LOAD_CONST_SMALL_INT 1 +393 RAISE_OBJ +394 LOAD_CONST_NONE +395 RETURN_VALUE +396 LOAD_CONST_SMALL_INT 1 +397 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 59 bytes) +Raw bytecode (code_info_size=8, bytecode_size=51): + a8 10 0a 02 80 82 34 38 81 57 c0 57 c1 57 c2 57 + c3 57 c4 57 c5 57 c6 57 c7 57 c8 c9 82 57 ca 57 + cb 57 cc 57 cd 57 ce 57 cf 57 26 10 57 26 11 57 + 26 12 26 13 b9 24 13 f2 59 51 63 +arg names: (N_STATE 22) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=133 + bc=0 line=131 + bc=20 line=132 + bc=44 line=133 00 LOAD_CONST_SMALL_INT 1 01 DUP_TOP 02 STORE_FAST 0 @@ -367,28 +464,29 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): 48 POP_TOP 49 LOAD_CONST_NONE 50 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +Raw bytecode (code_info_size=9, bytecode_size=11): + a1 01 0b 02 06 80 88 40 00 82 2a 01 53 b0 21 00 + 01 c1 51 63 arg names: a (N_STATE 5) (N_EXC_STACK 0) (INIT_CELL 0) -######## - bc=\\d\+ line=139 + bc=0 line=1 + bc=0 line=137 + bc=0 line=139 00 LOAD_CONST_SMALL_INT 2 01 BUILD_TUPLE 1 03 LOAD_NULL 04 LOAD_FAST 0 05 MAKE_CLOSURE_DEFARGS \.\+ 1 -\\d\+ STORE_FAST 1 -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +08 STORE_FAST 1 +09 LOAD_CONST_NONE +10 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 21 bytes) +Raw bytecode (code_info_size=8, bytecode_size=13): + 88 40 0a 02 80 8f 23 23 51 67 59 81 67 59 81 5e + 51 68 59 51 63 arg names: (N_STATE 2) (N_EXC_STACK 0) @@ -409,49 +507,47 @@ arg names: 10 POP_TOP 11 LOAD_CONST_NONE 12 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'Class' (descriptor: \.\+, bytecode @\.\+ 1\[56\] bytes) +Raw bytecode (code_info_size=\[56\], bytecode_size=10): + 00 \.\+ 11 0c 16 0d 10 03 16 0e 51 63 arg names: (N_STATE 1) (N_EXC_STACK 0) bc=0 line=1 ######## - bc=12 line=150 + bc=8 line=150 00 LOAD_NAME __name__ -03 STORE_NAME __module__ -06 LOAD_CONST_STRING 'Class' -09 STORE_NAME __qualname__ -12 LOAD_CONST_NONE -13 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +02 STORE_NAME __module__ +04 LOAD_CONST_STRING 'Class' +06 STORE_NAME __qualname__ +08 LOAD_CONST_NONE +09 RETURN_VALUE +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 18 bytes) +Raw bytecode (code_info_size=6, bytecode_size=12): + 19 08 02 0f 80 9c 12 10 12 11 b0 15 02 36 00 59 + 51 63 arg names: self (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=157 00 LOAD_GLOBAL super -\\d\+ LOAD_GLOBAL __class__ -\\d\+ LOAD_FAST 0 -\\d\+ LOAD_SUPER_METHOD f -\\d\+ CALL_METHOD n=0 nkw=0 -\\d\+ POP_TOP -\\d\+ LOAD_CONST_NONE -\\d\+ RETURN_VALUE -File cmdline/cmd_showbc.py, code block '<genexpr>' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +02 LOAD_GLOBAL __class__ +04 LOAD_FAST 0 +05 LOAD_SUPER_METHOD f +07 CALL_METHOD n=0 nkw=0 +09 POP_TOP +10 LOAD_CONST_NONE +11 RETURN_VALUE +File cmdline/cmd_showbc.py, code block '<genexpr>' (descriptor: \.\+, bytecode @\.\+ 31 bytes) +Raw bytecode (code_info_size=9, bytecode_size=22): + c3 40 0c 12 04 04 04 80 3b 53 b2 53 53 4b 0d 00 + c3 25 01 44 f7 7f 25 00 67 59 42 f0 7f 51 63 arg names: * * * (N_STATE 9) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=60 -######## 00 LOAD_NULL 01 LOAD_FAST 2 02 LOAD_NULL @@ -466,16 +562,15 @@ arg names: * * * 17 JUMP 4 20 LOAD_CONST_NONE 21 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '<listcomp>' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '<listcomp>' (descriptor: \.\+, bytecode @\.\+ 29 bytes) +Raw bytecode (code_info_size=8, bytecode_size=21): + 4b 0c 14 04 04 04 80 3c 2b 00 b2 5f 4b 0d 00 c3 + 25 01 44 f7 7f 25 00 2f 14 42 f0 7f 63 arg names: * * * (N_STATE 10) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=61 -######## 00 BUILD_LIST 0 02 LOAD_FAST 2 03 GET_ITER_STACK @@ -487,15 +582,15 @@ arg names: * * * 15 STORE_COMP 20 17 JUMP 4 20 RETURN_VALUE -File cmdline/cmd_showbc.py, code block '<dictcomp>' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block '<dictcomp>' (descriptor: \.\+, bytecode @\.\+ 31 bytes) +Raw bytecode (code_info_size=8, bytecode_size=23): + 53 0c 15 04 04 04 80 3d 2c 00 b2 5f 4b 0f 00 c3 + 25 01 44 f7 7f 25 00 25 00 2f 19 42 ee 7f 63 arg names: * * * (N_STATE 11) (N_EXC_STACK 0) bc=0 line=1 -######## + bc=0 line=62 00 BUILD_MAP 0 02 LOAD_FAST 2 03 GET_ITER_STACK @@ -508,16 +603,17 @@ arg names: * * * 17 STORE_COMP 25 19 JUMP 4 22 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'closure' (descriptor: \.\+, bytecode @\.\+ 20 bytes) +Raw bytecode (code_info_size=8, bytecode_size=12): + 19 0c 16 04 80 6f 25 23 25 00 81 f2 c1 81 27 00 + 29 00 51 63 arg names: * (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=114 + bc=0 line=112 + bc=5 line=113 + bc=8 line=114 00 LOAD_DEREF 0 02 LOAD_CONST_SMALL_INT 1 03 BINARY_OP 27 __add__ @@ -527,16 +623,14 @@ arg names: * 08 DELETE_DEREF 0 10 LOAD_CONST_NONE 11 RETURN_VALUE -File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): -######## -\.\+63 +File cmdline/cmd_showbc.py, code block 'f' (descriptor: \.\+, bytecode @\.\+ 13 bytes) +Raw bytecode (code_info_size=8, bytecode_size=5): + 9a 01 0a 02 04 09 80 8b b1 25 00 f2 63 arg names: * b (N_STATE 4) (N_EXC_STACK 0) bc=0 line=1 -######## - bc=\\d\+ line=140 + bc=0 line=140 00 LOAD_FAST 1 01 LOAD_DEREF 0 03 BINARY_OP 27 __add__ diff --git a/tests/cmdline/cmd_verbose.py.exp b/tests/cmdline/cmd_verbose.py.exp index 0edd050c22..ae833dbec8 100644 --- a/tests/cmdline/cmd_verbose.py.exp +++ b/tests/cmdline/cmd_verbose.py.exp @@ -1,19 +1,17 @@ -File cmdline/cmd_verbose.py, code block '<module>' (descriptor: \.\+, bytecode \.\+ bytes) -Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+): - 08 \.\+ -######## -\.\+63 +File cmdline/cmd_verbose.py, code block '<module>' (descriptor: \.\+, bytecode @\.\+ 12 bytes) +Raw bytecode (code_info_size=4, bytecode_size=8): + 08 04 01 40 11 02 81 34 01 59 51 63 arg names: (N_STATE 2) (N_EXC_STACK 0) bc=0 line=1 bc=0 line=3 00 LOAD_NAME print -03 LOAD_CONST_SMALL_INT 1 -04 CALL_FUNCTION n=1 nkw=0 -06 POP_TOP -07 LOAD_CONST_NONE -08 RETURN_VALUE +02 LOAD_CONST_SMALL_INT 1 +03 CALL_FUNCTION n=1 nkw=0 +05 POP_TOP +06 LOAD_CONST_NONE +07 RETURN_VALUE 1 mem: total=\\d\+, current=\\d\+, peak=\\d\+ stack: \\d\+ out of \\d\+ diff --git a/tests/micropython/import_mpy_invalid.py b/tests/micropython/import_mpy_invalid.py index 02fd4b1254..b02312a7a8 100644 --- a/tests/micropython/import_mpy_invalid.py +++ b/tests/micropython/import_mpy_invalid.py @@ -49,7 +49,6 @@ user_files = { "/mod0.mpy": b"", # empty file "/mod1.mpy": b"M", # too short header "/mod2.mpy": b"M\x00\x00\x00", # bad version - "/mod3.mpy": b"M\x00\x00\x00\x7f", # qstr window too large } # create and mount a user filesystem diff --git a/tests/micropython/import_mpy_invalid.py.exp b/tests/micropython/import_mpy_invalid.py.exp index ea748ff6c1..1727ea1cea 100644 --- a/tests/micropython/import_mpy_invalid.py.exp +++ b/tests/micropython/import_mpy_invalid.py.exp @@ -1,4 +1,3 @@ mod0 ValueError incompatible .mpy file mod1 ValueError incompatible .mpy file mod2 ValueError incompatible .mpy file -mod3 ValueError incompatible .mpy file diff --git a/tests/micropython/import_mpy_native_gc.py b/tests/micropython/import_mpy_native_gc.py index f982a4ee17..6f37b15ab7 100644 --- a/tests/micropython/import_mpy_native_gc.py +++ b/tests/micropython/import_mpy_native_gc.py @@ -49,14 +49,14 @@ class UserFS: # by the required value of sys.implementation.mpy. features0_file_contents = { # -march=x64 - 0xA05: b'M\x05\x0a\x1f \x84b\xe9/\x00\x00\x00SH\x8b\x1ds\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dA\x00\x00\x00H\x8b\x7f\x08L\x8bc(A\xff\xd4H\x8d5\x1f\x00\x00\x00H\x89\xc5H\x8b\x05-\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x90\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x84@\x12factorial\x10\x00\x00\r \x01"\x9f\x1c\x01\x1e\xff', + 0xA06: b'M\x06\n\x1f\x01\x004build/features0.native.mpy\x00\x8aB\xe9/\x00\x00\x00SH\x8b\x1d\x83\x00\x00\x00\xbe\x02\x00\x00\x00\xffS\x18\xbf\x01\x00\x00\x00H\x85\xc0u\x0cH\x8bC \xbe\x02\x00\x00\x00[\xff\xe0H\x0f\xaf\xf8H\xff\xc8\xeb\xe6ATUSH\x8b\x1dQ\x00\x00\x00H\x8bG\x08L\x8bc(H\x8bx\x08A\xff\xd4H\x8d5+\x00\x00\x00H\x89\xc5H\x8b\x059\x00\x00\x00\x0f\xb78\xffShH\x89\xefA\xff\xd4H\x8b\x03[]A\\\xc3\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xa0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x85\x00\x12factorial\x00\x10\r$\x01&\x9f \x01"\xff', # -march=armv7m - 0x1605: b"M\x05\x16\x1f \x84\x12\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfj\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\tN\tK~D\xf4X@hgi\xb8G\x05F\x07K\x07I\xf2XyD\x10\x88ck\x98G(F\xb8G h\xf8\xbd6\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1c\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x80\x00\x00\x00\x00\x00\x00\x00\x01\x84\x00\x12factorial\x10\x00\x00\r<\x01>\x9f8\x01:\xff", + 0x1606: b"M\x06\x16\x1f\x01\x004build/features0.native.mpy\x00\x88B\x1a\xe0\x00\x00\x13\xb5\nK\nJ{D\x9cX\x02!\xe3h\x98G\x03F\x01 3\xb9\x02!#i\x01\x93\x02\xb0\xbd\xe8\x10@\x18GXC\x01;\xf4\xe7\x00\xbfn\x00\x00\x00\x00\x00\x00\x00\xf8\xb5\nN\nK~D\xf4XChgiXh\xb8G\x05F\x07K\x08I\xf2XyD\x10\x88ck\x98G(F\xb8G h\xf8\xbd\x00\xbf:\x00\x00\x00\x00\x00\x00\x00\x04\x00\x00\x00\x1e\x00\x00\x00\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x84\x00\x00\x00\x00\x00\x00\x00\x01\x84\x10\x12factorial\x00\x10\r>\x01@\x9f:\x01<\xff", } # Populate other armv7m-derived archs based on armv7m. -for arch in (0x1A05, 0x1E05, 0x2205): - features0_file_contents[arch] = features0_file_contents[0x1605] +for arch in (0x1A06, 0x1E06, 0x2206): + features0_file_contents[arch] = features0_file_contents[0x1606] if sys.implementation.mpy not in features0_file_contents: print("SKIP") diff --git a/tests/micropython/import_mpy_native_x64.py b/tests/micropython/import_mpy_native_x64.py index ff0142a9cf..3a25df1866 100644 --- a/tests/micropython/import_mpy_native_x64.py +++ b/tests/micropython/import_mpy_native_x64.py @@ -52,49 +52,62 @@ class UserFS: # fmt: off user_files = { # bad architecture - '/mod0.mpy': b'M\x05\xfe\x00\x10', + '/mod0.mpy': b'M\x06\xfe\x00\x10', # test loading of viper and asm '/mod1.mpy': ( - b'M\x05\x0a\x1f\x20' # header + b'M\x06\x0a\x1f' # header - b'\x20' # n bytes, bytecode - b'\x00\x08\x02m\x02m' # prelude + b'\x02' # n_qstr + b'\x00' # n_obj + + b'\x0emod1.py\x00' # qstr0 = "mod1.py" + b'\x0aouter\x00' # qstr1 = "outer" + + b'\x2c' # 5 bytes, have children, bytecode + b'\x00\x02' # prelude + b'\x01' # simple name (qstr index) b'\x51' # LOAD_CONST_NONE b'\x63' # RETURN_VALUE - b'\x00\x02' # n_obj, n_raw_code + b'\x02' # 2 children - b'\x22' # n bytes, viper code - b'\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00' # qstr0 - b'\x01\x0c\x0aprint' # n_qstr, qstr0 - b'\x00\x00\x00' # scope_flags, n_obj, n_raw_code + b'\x42' # 8 bytes, no children, viper code + b'\x00\x00\x00\x00\x00\x00' # dummy machine code + b'\x00\x00' # slot for qstr0 + b'\x01\x0c\x0aprint\x00' # n_qstr=1, qstr0 + b'\x00' # scope_flags - b'\x23' # n bytes, asm code - b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code - b'\x00\x00\x00' # scope_flags, n_pos_args, type_sig + b'\x43' # 8 bytes, no children, asm code + b'\x00\x00\x00\x00\x00\x00\x00\x00' # dummy machine code + b'\x00\x00\x00' # scope_flags, n_pos_args, type_sig ), # test loading viper with additional scope flags and relocation '/mod2.mpy': ( - b'M\x05\x0a\x1f\x20' # header + b'M\x06\x0a\x1f' # header + + b'\x02' # n_qstr + b'\x00' # n_obj + + b'\x0emod2.py\x00' # qstr0 = "mod2.py" + b'\x0aouter\x00' # qstr1 = "outer" - b'\x20' # n bytes, bytecode - b'\x00\x08\x02m\x02m' # prelude + b'\x2c' # 5 bytes, have children, bytecode + b'\x00\x02' # prelude + b'\x01' # simple name (qstr index) b'\x51' # LOAD_CONST_NONE b'\x63' # RETURN_VALUE - b'\x00\x01' # n_obj, n_raw_code + b'\x01' # 1 child - b'\x12' # n bytes(=4), viper code - b'\x00\x00\x00\x00' # dummy machine code - b'\x00' # n_qstr - b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC - b'\x00\x00' # n_obj, n_raw_code - b'\x06rodata' # rodata, 6 bytes - b'\x04' # bss, 4 bytes - b'\x03\x01\x00' # dummy relocation of rodata + b'\x22' # 4 bytes, no children, viper code + b'\x00\x00\x00\x00' # dummy machine code + b'\x00' # n_qstr=0 + b'\x70' # scope_flags: VIPERBSS | VIPERRODATA | VIPERRELOC + b'\x06\x04' # rodata=6 bytes, bss=4 bytes + b'rodata' # rodata content + b'\x03\x01\x00' # dummy relocation of rodata ), } # fmt: on diff --git a/tests/perf_bench/core_import_mpy_multi.py b/tests/perf_bench/core_import_mpy_multi.py index 0da466206b..74caf20a38 100644 --- a/tests/perf_bench/core_import_mpy_multi.py +++ b/tests/perf_bench/core_import_mpy_multi.py @@ -23,7 +23,7 @@ def f(): x = ("const tuple", None, False, True, 1, 2, 3) result = 123 """ -file_data = b'M\x05\x02\x1f \x81\x0c\x10\x12\x00\x07\x0etest.py\x8b\x07e`\x00T2\x00\x10\x02A4\x02\x16\x012\x01\x16\x02f"\x80{\x16\x0cresultQc\x00\x02\x81\x1c\x00\x10\x05\x07,EE\x00\x11\x00\x17\x16\x00\x16\x10\x03\x16\x00\x1a2\x00\x16\x00\x112\x01\x16\x00\xa42\x02\x16\x00}Qc\x00\x03<\x1a\x0c\x00\x11\x03@\x00\xb1\xb0\x18\x06argQc\x00\x00\x00\x89\x01,\t\x0e\x00\xa4\x03` \x00Qc\x00\x00\x00\x89,\t\x0e\x00}\x01``\x00Qc\x00\x00\x00\x89\x81l8\x16\t\x03\x80\x08/,##\x00\x12\x00{\x12\x00\x97\x12\x00B\x12\x00K*\x04Y\x12\x00$\x12\x007\x12\x006*\x03Y#\x00\xc0#\x01\xc0#\x02QPR\x81\x82\x83*\x07\xc0Qc\x03\x00s\x1cthis will be a string objectb\x1bthis will be a bytes objects\x0bconst tuple' +file_data = b'M\x06\x02\x1f\x14\x03\x0etest.py\x00\x0f\x02A\x00\x02f\x00\x0cresult\x00/-5#\x82I\x81{\x81w\x82/\x81\x05\x81\x17Iom\x82\x13\x06arg\x00s\x1cthis will be a string object\x00b\x1bthis will be a bytes object\x00s\x0bconst tuple\x00\x81\\\x10\n\x01\x89\x07d`T2\x00\x10\x024\x02\x16\x022\x01\x16\x03"\x80{\x16\x04Qc\x02\x81d\x00\x08\x02(DD\x11\x05\x16\x06\x10\x02\x16\x072\x00\x16\x082\x01\x16\t2\x02\x16\nQc\x03`\x1a\x08\x08\x12\x13@\xb1\xb0\x18\x13Qc@\t\x08\t\x12` Qc@\t\x08\n\x12``Qc\x83\x008\x0e\x03\x80\x08+)##\x12\x0b\x12\x0c\x12\r\x12\x0e*\x04Y\x12\x0f\x12\x10\x12\x11*\x03Y#\x00\xc0#\x01\xc0#\x02QPR\x81\x82\x83*\x07\xc0Qc' class File(uio.IOBase): diff --git a/tests/perf_bench/core_import_mpy_single.py b/tests/perf_bench/core_import_mpy_single.py index 5ca3584959..1e8a4f531e 100644 --- a/tests/perf_bench/core_import_mpy_single.py +++ b/tests/perf_bench/core_import_mpy_single.py @@ -78,7 +78,7 @@ def f1(): x = ("const tuple 9", None, False, True, 1, 2, 3) result = 123 """ -file_data = b'M\x05\x02\x1f \x81\\\x10\x1a\x00\x07\x0etest.pyk@k@\x85\x18\x85\x1f\x00T2\x00\x10\x04A04\x02\x16\x01T2\x01\x10\x04A14\x02\x16\x012\x02\x16\x04f02\x03\x16\x04f1"\x80{\x16\x0cresultQc\x00\x04\x814\x00\x12\t\x0b,%%%\x00\x11\x00\x17\x16\x00\x16\x10\x03\x16\x00\x1a2\x00\x16\x04a02\x01\x16\x04a12\x02\x16\x04a22\x03\x16\x04a3Qc\x00\x04(\t\x0c\x07\x0b \x00Qc\x00\x00\x00\x89(\t\x0c\t\x03@\x00Qc\x00\x00\x00\x89(\t\x0c\t\x03`\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03` \x00Qc\x00\x00\x00\x89\x818\x00\x14\x13\x03l`%%%\x00\x11\x00\x17\x16\x00\x16\x10\x03\x16\x00\x1a2\x00\x16\x0b2\x01\x16\x0b2\x02\x16\x0b2\x03\x16\x0bQc\x00\x04,\t\x0e\x07\x0b``\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03\x80\x07\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03\x80\x08\x00Qc\x00\x00\x00\x89,\t\x0e\t\x03\x80\t\x00Qc\x00\x00\x00\x89\x94\x1cPP\x13\x03\x80\x0b8;555222\x1f%\x1f%\x1f"\x1f"\x1f%\x1f%\x1f"\x1f%\x1f"\x1f%)\x1f"//\x00\x12\x00\t\x12\x00\n\x12\x00\x0b\x12\x00\x0c\x12\x00\r\x12\x00\x0e\x12\x00\x0f*\x07Y\x12\x00\x10\x12\x00\x11\x12\x00\x12\x12\x00\x13\x12\x00\x14\x12\x00\x15\x12\x00\x16\x12\x00\x17*\x08Y\x12\x00\x18\x12\x00\x19\x12\x00\x1a\x12\x00\x1b\x12\x00\x1c\x12\x00\x1d*\x06Y\x12\x00\x1e\x12\x00\x1f\x12\x00 \x12\x00!\x12\x00"\x12\x00#*\x06Y\x12\x00$\x12\x00%\x12\x00&\x12\x00\'\x12\x00(\x12\x00)*\x06Y\x12\x00*\x12\x00+\x12\x00,\x12\x00-\x12\x00.*\x05Y\x12\x00/\x12\x000\x12\x001\x12\x002\x12\x003*\x05Y\x12\x004\x12\x005\x12\x006\x12\x007\x12\x008*\x05Y\x12\x009\x12\x00:\x12\x00;\x12\x00<\x12\x00=\x12\x00>\x12\x00?\x12\x00@\x12\x00A\x12\x00B\x12\x00C*\x0bY\x12\x00D\x12\x00E\x12\x00F\x12\x00G\x12\x00H\x12\x00I\x12\x00J\x12\x00K\x12\x00L\x12\x00M\x12\x00N*\x0bY\x12\x00O\x12\x00P\x12\x00Q\x12\x00R\x12\x00S\x12\x00T\x12\x00U\x12\x00V\x12\x00W\x12\x00X*\nY\x12\x00Y\x12\x00Z\x12\x00[\x12\x00\\\x12\x00]\x12\x00^\x12\x00_\x12\x00`\x12\x00a\x12\x00b*\nY\x12\x00c\x12\x00d\x12\x00e\x12\x00f\x12\x00g\x12\x00h\x12\x00i\x12\x00j\x12\x00k\x12\x00l\x12\x00m*\x0bY\x12\x00n\x12\x00o\x12\x00p\x12\x00q\x12\x00r\x12\x00s\x12\x00t\x12\x00u\x12\x00v\x12\x00w\x12\x00x*\x0bY\x12\x00y\x12\x00z\x12\x00{\x12\x00|\x12\x00}\x12\x00~\x12\x00\x7f\x12\x00\x80\x12\x00\x81\x12\x00\x82*\nY\x12\x00\x83\x12\x00\x84\x12\x00\x85\x12\x00\x86\x12\x00\x87\x12\x00\x88\x12\x00\x89\x12\x00\x8a\x12\x00\x8b\x12\x00\x8c\x12\x00\x8d*\x0bY\x12\x00\x8e\x12\x00\x8f\x12\x00\x90\x12\x00\x91\x12\x00\x92\x12\x00\x93\x12\x00\x94\x12\x00\x95\x12\x00\x96\x12\x00\x97*\nY\x12\x00\x98\x12\x00\x99\x12\x00\x9a\x12\x00\x9b\x12\x00\x9c\x12\x00\x9d\x12\x00\x9e\x12\x00\x9f\x12\x00\xa0\x12\x00\xa2\x12\x00\xa3*\x0bY\x12\x00\xa4\x12\x00\xa5*\x02Y\x12\nname0\x12\nname1\x12\nname2\x12\nname3\x12\nname4\x12\nname5\x12\nname6\x12\nname7\x12\nname8\x12\nname9*\nY\x12$quite_a_long_name0\x12$quite_a_long_name1\x12$quite_a_long_name2\x12$quite_a_long_name3*\x04Y\x12$quite_a_long_name4\x12$quite_a_long_name5\x12$quite_a_long_name6\x12$quite_a_long_name7*\x04Y\x12$quite_a_long_name8\x12$quite_a_long_name9\x12&quite_a_long_name10\x12&quite_a_long_name11*\x04YQc\x00\x00\x86H8H?/\x80#####################+++++++++\x00#\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14QPR\x81\x82\x83*\x07\xc0#\x15QPR\x81\x82\x83*\x07\xc0#\x16QPR\x81\x82\x83*\x07\xc0#\x17QPR\x81\x82\x83*\x07\xc0#\x18QPR\x81\x82\x83*\x07\xc0#\x19QPR\x81\x82\x83*\x07\xc0#\x1aQPR\x81\x82\x83*\x07\xc0#\x1bQPR\x81\x82\x83*\x07\xc0#\x1cQPR\x81\x82\x83*\x07\xc0#\x1dQPR\x81\x82\x83*\x07\xc0Qc\x1e\x00s\x1ethis will be a string object 0s\x1ethis will be a string object 1s\x1ethis will be a string object 2s\x1ethis will be a string object 3s\x1ethis will be a string object 4s\x1ethis will be a string object 5s\x1ethis will be a string object 6s\x1ethis will be a string object 7s\x1ethis will be a string object 8s\x1ethis will be a string object 9b\x1dthis will be a bytes object 0b\x1dthis will be a bytes object 1b\x1dthis will be a bytes object 2b\x1dthis will be a bytes object 3b\x1dthis will be a bytes object 4b\x1dthis will be a bytes object 5b\x1dthis will be a bytes object 6b\x1dthis will be a bytes object 7b\x1dthis will be a bytes object 8b\x1dthis will be a bytes object 9s\rconst tuple 0s\rconst tuple 1s\rconst tuple 2s\rconst tuple 3s\rconst tuple 4s\rconst tuple 5s\rconst tuple 6s\rconst tuple 7s\rconst tuple 8s\rconst tuple 9' +file_data = b"M\x06\x02\x1f\x81=\x1e\x0etest.py\x00\x0f\x04A0\x00\x04A1\x00\x04f0\x00\x04f1\x00\x0cresult\x00/-5\x04a0\x00\x04a1\x00\x04a2\x00\x04a3\x00\x13\x15\x17\x19\x1b\x1d\x1f!#%')+1379;=?ACEGIKMOQSUWY[]_acegikmoqsuwy{}\x7f\x81\x01\x81\x03\x81\x05\x81\x07\x81\t\x81\x0b\x81\r\x81\x0f\x81\x11\x81\x13\x81\x15\x81\x17\x81\x19\x81\x1b\x81\x1d\x81\x1f\x81!\x81#\x81%\x81'\x81)\x81+\x81-\x81/\x811\x813\x815\x817\x819\x81;\x81=\x81?\x81A\x81C\x81E\x81G\x81I\x81K\x81M\x81O\x81Q\x81S\x81U\x81W\x81Y\x81[\x81]\x81_\x81a\x81c\x81e\x81g\x81i\x81k\x81m\x81o\x81q\x81s\x81u\x81w\x81y\x81{\x81}\x81\x7f\x82\x01\x82\x03\x82\x05\x82\x07\x82\t\x82\x0b\x82\r\x82\x0f\x82\x11\x82\x13\x82\x15\x82\x17\x82\x19\x82\x1b\x82\x1d\x82\x1f\x82!\x82#\x82%\x82'\x82)\x82+\x82-\x82/\x821\x823\x825\x827\x829\x82;\x82=\x82?\x82A\x82E\x82G\x82I\x82K\nname0\x00\nname1\x00\nname2\x00\nname3\x00\nname4\x00\nname5\x00\nname6\x00\nname7\x00\nname8\x00\nname9\x00$quite_a_long_name0\x00$quite_a_long_name1\x00$quite_a_long_name2\x00$quite_a_long_name3\x00$quite_a_long_name4\x00$quite_a_long_name5\x00$quite_a_long_name6\x00$quite_a_long_name7\x00$quite_a_long_name8\x00$quite_a_long_name9\x00&quite_a_long_name10\x00&quite_a_long_name11\x00s\x1ethis will be a string object 0\x00s\x1ethis will be a string object 1\x00s\x1ethis will be a string object 2\x00s\x1ethis will be a string object 3\x00s\x1ethis will be a string object 4\x00s\x1ethis will be a string object 5\x00s\x1ethis will be a string object 6\x00s\x1ethis will be a string object 7\x00s\x1ethis will be a string object 8\x00s\x1ethis will be a string object 9\x00b\x1dthis will be a bytes object 0\x00b\x1dthis will be a bytes object 1\x00b\x1dthis will be a bytes object 2\x00b\x1dthis will be a bytes object 3\x00b\x1dthis will be a bytes object 4\x00b\x1dthis will be a bytes object 5\x00b\x1dthis will be a bytes object 6\x00b\x1dthis will be a bytes object 7\x00b\x1dthis will be a bytes object 8\x00b\x1dthis will be a bytes object 9\x00s\rconst tuple 0\x00s\rconst tuple 1\x00s\rconst tuple 2\x00s\rconst tuple 3\x00s\rconst tuple 4\x00s\rconst tuple 5\x00s\rconst tuple 6\x00s\rconst tuple 7\x00s\rconst tuple 8\x00s\rconst tuple 9\x00\x82d\x10\x12\x01i@i@\x84\x18\x84\x1fT2\x00\x10\x024\x02\x16\x02T2\x01\x10\x034\x02\x16\x032\x02\x16\x042\x03\x16\x05\"\x80{\x16\x06Qc\x04\x82\x0c\x00\n\x02($$$\x11\x07\x16\x08\x10\x02\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04@\t\x08\n\x81\x0b Qc@\t\x08\x0b\x81\x0b@Qc@\t\x08\x0c\x81\x0b`QcH\t\n\r\x81\x0b` Qc\x82\x14\x00\x0c\x03h`$$$\x11\x07\x16\x08\x10\x03\x16\t2\x00\x16\n2\x01\x16\x0b2\x02\x16\x0c2\x03\x16\rQc\x04H\t\n\n\x81\x0b``QcH\t\n\x0b\x81\x0b\x80\x07QcH\t\n\x0c\x81\x0b\x80\x08QcH\t\n\r\x81\x0b\x80\tQc\xa08P:\x04\x80\x0b13///---997799<\x1f%\x1f\"\x1f%)\x1f\"//\x12\x0e\x12\x0f\x12\x10\x12\x11\x12\x12\x12\x13\x12\x14*\x07Y\x12\x15\x12\x16\x12\x17\x12\x18\x12\x19\x12\x1a\x12\x08\x12\x07*\x08Y\x12\x1b\x12\x1c\x12\t\x12\x1d\x12\x1e\x12\x1f*\x06Y\x12 \x12!\x12\"\x12#\x12$\x12%*\x06Y\x12&\x12'\x12(\x12)\x12*\x12+*\x06Y\x12,\x12-\x12.\x12/\x120*\x05Y\x121\x122\x123\x124\x125*\x05Y\x126\x127\x128\x129\x12:*\x05Y\x12;\x12<\x12=\x12>\x12?\x12@\x12A\x12B\x12C\x12D\x12E*\x0bY\x12F\x12G\x12H\x12I\x12J\x12K\x12L\x12M\x12N\x12O\x12P*\x0bY\x12Q\x12R\x12S\x12T\x12U\x12V\x12W\x12X\x12Y\x12Z*\nY\x12[\x12\\\x12]\x12^\x12_\x12`\x12a\x12b\x12c\x12d*\nY\x12e\x12f\x12g\x12h\x12i\x12j\x12k\x12l\x12m\x12n\x12o*\x0bY\x12p\x12q\x12r\x12s\x12t\x12u\x12v\x12w\x12x\x12y\x12z*\x0bY\x12{\x12|\x12}\x12~\x12\x7f\x12\x81\x00\x12\x81\x01\x12\x81\x02\x12\x81\x03\x12\x81\x04*\nY\x12\x81\x05\x12\x81\x06\x12\x81\x07\x12\x81\x08\x12\x81\t\x12\x81\n\x12\x81\x0b\x12\x81\x0c\x12\x81\r\x12\x81\x0e\x12\x81\x0f*\x0bY\x12\x81\x10\x12\x81\x11\x12\x81\x12\x12\x81\x13\x12\x81\x14\x12\x81\x15\x12\x81\x16\x12\x81\x17\x12\x81\x18\x12\x81\x19*\nY\x12\x81\x1a\x12\x81\x1b\x12\x81\x1c\x12\x81\x1d\x12\x81\x1e\x12\x81\x1f\x12\x81 \x12\x81!\x12\x81\"\x12\x81#\x12\x81$*\x0bY\x12\x81%\x12\x81&*\x02Y\x12\x81'\x12\x81(\x12\x81)\x12\x81*\x12\x81+\x12\x81,\x12\x81-\x12\x81.\x12\x81/\x12\x810*\nY\x12\x811\x12\x812\x12\x813\x12\x814*\x04Y\x12\x815\x12\x816\x12\x817\x12\x818*\x04Y\x12\x819\x12\x81:\x12\x81;\x12\x81<*\x04YQc\x8cp8@\x05\x80#####################+++++++++#\x00\xc0#\x01\xc0#\x02\xc0#\x03\xc0#\x04\xc0#\x05\xc0#\x06\xc0#\x07\xc0#\x08\xc0#\t\xc0#\n\xc0#\x0b\xc0#\x0c\xc0#\r\xc0#\x0e\xc0#\x0f\xc0#\x10\xc0#\x11\xc0#\x12\xc0#\x13\xc0#\x14QPR\x81\x82\x83*\x07\xc0#\x15QPR\x81\x82\x83*\x07\xc0#\x16QPR\x81\x82\x83*\x07\xc0#\x17QPR\x81\x82\x83*\x07\xc0#\x18QPR\x81\x82\x83*\x07\xc0#\x19QPR\x81\x82\x83*\x07\xc0#\x1aQPR\x81\x82\x83*\x07\xc0#\x1bQPR\x81\x82\x83*\x07\xc0#\x1cQPR\x81\x82\x83*\x07\xc0#\x1dQPR\x81\x82\x83*\x07\xc0Qc" class File(uio.IOBase): diff --git a/tools/mpy-tool.py b/tools/mpy-tool.py index 58a4a75c91..da6352dab7 100755 --- a/tools/mpy-tool.py +++ b/tools/mpy-tool.py @@ -29,27 +29,49 @@ from __future__ import print_function import platform if platform.python_version_tuple()[0] == "2": - str_cons = lambda val, enc=None: val + from binascii import hexlify as hexlify_py2 + + str_cons = lambda val, enc=None: str(val) bytes_cons = lambda val, enc=None: bytearray(val) is_str_type = lambda o: type(o) is str is_bytes_type = lambda o: type(o) is bytearray is_int_type = lambda o: type(o) is int or type(o) is long + + def hexlify_to_str(b): + x = hexlify_py2(b) + return ":".join(x[i : i + 2] for i in range(0, len(x), 2)) + else: + from binascii import hexlify + str_cons = str bytes_cons = bytes is_str_type = lambda o: type(o) is str is_bytes_type = lambda o: type(o) is bytes is_int_type = lambda o: type(o) is int + + def hexlify_to_str(b): + return str(hexlify(b, ":"), "ascii") + + # end compatibility code import sys import struct -from collections import namedtuple sys.path.append(sys.path[0] + "/../py") import makeqstrdata as qstrutil +class MPYReadError(Exception): + def __init__(self, filename, msg): + self.filename = filename + self.msg = msg + + def __str__(self): + return "%s: %s" % (self.filename, self.msg) + + class FreezeError(Exception): def __init__(self, rawcode, msg): self.rawcode = rawcode @@ -60,7 +82,7 @@ class FreezeError(Exception): class Config: - MPY_VERSION = 5 + MPY_VERSION = 6 MICROPY_LONGINT_IMPL_NONE = 0 MICROPY_LONGINT_IMPL_LONGLONG = 1 MICROPY_LONGINT_IMPL_MPZ = 2 @@ -82,20 +104,6 @@ for n in qstrutil.static_qstr_list: global_qstrs.append(QStrType(n)) -class QStrWindow: - def __init__(self, size): - self.window = [] - self.size = size - - def push(self, val): - self.window = [val] + self.window[: self.size - 1] - - def access(self, idx): - val = self.window[idx] - self.window = [val] + self.window[:idx] + self.window[idx + 1 :] - return val - - MP_CODE_BYTECODE = 2 MP_CODE_NATIVE_PY = 3 MP_CODE_NATIVE_VIPER = 4 @@ -113,6 +121,10 @@ MP_NATIVE_ARCH_ARMV7EMDP = 8 MP_NATIVE_ARCH_XTENSA = 9 MP_NATIVE_ARCH_XTENSAWIN = 10 +MP_SCOPE_FLAG_VIPERRELOC = 0x10 +MP_SCOPE_FLAG_VIPERRODATA = 0x20 +MP_SCOPE_FLAG_VIPERBSS = 0x40 + MP_BC_MASK_EXTRA_BYTE = 0x9E MP_BC_FORMAT_BYTE = 0 @@ -120,11 +132,180 @@ MP_BC_FORMAT_QSTR = 1 MP_BC_FORMAT_VAR_UINT = 2 MP_BC_FORMAT_OFFSET = 3 -# extra byte if caching enabled: -MP_BC_LOAD_NAME = 0x11 -MP_BC_LOAD_GLOBAL = 0x12 -MP_BC_LOAD_ATTR = 0x13 -MP_BC_STORE_ATTR = 0x18 +mp_unary_op_method_name = ( + "__pos__", + "__neg__", + "__invert__", + "<not>", +) + +mp_binary_op_method_name = ( + "__lt__", + "__gt__", + "__eq__", + "__le__", + "__ge__", + "__ne__", + "<in>", + "<is>", + "<exception match>", + "__ior__", + "__ixor__", + "__iand__", + "__ilshift__", + "__irshift__", + "__iadd__", + "__isub__", + "__imul__", + "__imatmul__", + "__ifloordiv__", + "__itruediv__", + "__imod__", + "__ipow__", + "__or__", + "__xor__", + "__and__", + "__lshift__", + "__rshift__", + "__add__", + "__sub__", + "__mul__", + "__matmul__", + "__floordiv__", + "__truediv__", + "__mod__", + "__pow__", +) + + +class Opcodes: + # fmt: off + # Load, Store, Delete, Import, Make, Build, Unpack, Call, Jump, Exception, For, sTack, Return, Yield, Op + MP_BC_BASE_RESERVED = (0x00) # ---------------- + MP_BC_BASE_QSTR_O = (0x10) # LLLLLLSSSDDII--- + MP_BC_BASE_VINT_E = (0x20) # MMLLLLSSDDBBBBBB + MP_BC_BASE_VINT_O = (0x30) # UUMMCCCC-------- + MP_BC_BASE_JUMP_E = (0x40) # J-JJJJJEEEEF---- + MP_BC_BASE_BYTE_O = (0x50) # LLLLSSDTTTTTEEFF + MP_BC_BASE_BYTE_E = (0x60) # --BREEEYYI------ + MP_BC_LOAD_CONST_SMALL_INT_MULTI = (0x70) # LLLLLLLLLLLLLLLL + # = (0x80) # LLLLLLLLLLLLLLLL + # = (0x90) # LLLLLLLLLLLLLLLL + # = (0xa0) # LLLLLLLLLLLLLLLL + MP_BC_LOAD_FAST_MULTI = (0xb0) # LLLLLLLLLLLLLLLL + MP_BC_STORE_FAST_MULTI = (0xc0) # SSSSSSSSSSSSSSSS + MP_BC_UNARY_OP_MULTI = (0xd0) # OOOOOOO + MP_BC_BINARY_OP_MULTI = (0xd7) # OOOOOOOOO + # = (0xe0) # OOOOOOOOOOOOOOOO + # = (0xf0) # OOOOOOOOOO------ + + MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM = 64 + MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS = 16 + MP_BC_LOAD_FAST_MULTI_NUM = 16 + MP_BC_STORE_FAST_MULTI_NUM = 16 + MP_BC_UNARY_OP_MULTI_NUM = 4 # MP_UNARY_OP_NUM_BYTECODE + MP_BC_BINARY_OP_MULTI_NUM = 35 # MP_BINARY_OP_NUM_BYTECODE + + MP_BC_LOAD_CONST_FALSE = (MP_BC_BASE_BYTE_O + 0x00) + MP_BC_LOAD_CONST_NONE = (MP_BC_BASE_BYTE_O + 0x01) + MP_BC_LOAD_CONST_TRUE = (MP_BC_BASE_BYTE_O + 0x02) + MP_BC_LOAD_CONST_SMALL_INT = (MP_BC_BASE_VINT_E + 0x02) # signed var-int + MP_BC_LOAD_CONST_STRING = (MP_BC_BASE_QSTR_O + 0x00) # qstr + MP_BC_LOAD_CONST_OBJ = (MP_BC_BASE_VINT_E + 0x03) # ptr + MP_BC_LOAD_NULL = (MP_BC_BASE_BYTE_O + 0x03) + + MP_BC_LOAD_FAST_N = (MP_BC_BASE_VINT_E + 0x04) # uint + MP_BC_LOAD_DEREF = (MP_BC_BASE_VINT_E + 0x05) # uint + MP_BC_LOAD_NAME = (MP_BC_BASE_QSTR_O + 0x01) # qstr + MP_BC_LOAD_GLOBAL = (MP_BC_BASE_QSTR_O + 0x02) # qstr + MP_BC_LOAD_ATTR = (MP_BC_BASE_QSTR_O + 0x03) # qstr + MP_BC_LOAD_METHOD = (MP_BC_BASE_QSTR_O + 0x04) # qstr + MP_BC_LOAD_SUPER_METHOD = (MP_BC_BASE_QSTR_O + 0x05) # qstr + MP_BC_LOAD_BUILD_CLASS = (MP_BC_BASE_BYTE_O + 0x04) + MP_BC_LOAD_SUBSCR = (MP_BC_BASE_BYTE_O + 0x05) + + MP_BC_STORE_FAST_N = (MP_BC_BASE_VINT_E + 0x06) # uint + MP_BC_STORE_DEREF = (MP_BC_BASE_VINT_E + 0x07) # uint + MP_BC_STORE_NAME = (MP_BC_BASE_QSTR_O + 0x06) # qstr + MP_BC_STORE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x07) # qstr + MP_BC_STORE_ATTR = (MP_BC_BASE_QSTR_O + 0x08) # qstr + MP_BC_STORE_SUBSCR = (MP_BC_BASE_BYTE_O + 0x06) + + MP_BC_DELETE_FAST = (MP_BC_BASE_VINT_E + 0x08) # uint + MP_BC_DELETE_DEREF = (MP_BC_BASE_VINT_E + 0x09) # uint + MP_BC_DELETE_NAME = (MP_BC_BASE_QSTR_O + 0x09) # qstr + MP_BC_DELETE_GLOBAL = (MP_BC_BASE_QSTR_O + 0x0a) # qstr + + MP_BC_DUP_TOP = (MP_BC_BASE_BYTE_O + 0x07) + MP_BC_DUP_TOP_TWO = (MP_BC_BASE_BYTE_O + 0x08) + MP_BC_POP_TOP = (MP_BC_BASE_BYTE_O + 0x09) + MP_BC_ROT_TWO = (MP_BC_BASE_BYTE_O + 0x0a) + MP_BC_ROT_THREE = (MP_BC_BASE_BYTE_O + 0x0b) + + MP_BC_JUMP = (MP_BC_BASE_JUMP_E + 0x02) # rel byte code offset, 16-bit signed, in excess + MP_BC_POP_JUMP_IF_TRUE = (MP_BC_BASE_JUMP_E + 0x03) # rel byte code offset, 16-bit signed, in excess + MP_BC_POP_JUMP_IF_FALSE = (MP_BC_BASE_JUMP_E + 0x04) # rel byte code offset, 16-bit signed, in excess + MP_BC_JUMP_IF_TRUE_OR_POP = (MP_BC_BASE_JUMP_E + 0x05) # rel byte code offset, 16-bit signed, in excess + MP_BC_JUMP_IF_FALSE_OR_POP = (MP_BC_BASE_JUMP_E + 0x06) # rel byte code offset, 16-bit signed, in excess + MP_BC_UNWIND_JUMP = (MP_BC_BASE_JUMP_E + 0x00) # rel byte code offset, 16-bit signed, in excess; then a byte + MP_BC_SETUP_WITH = (MP_BC_BASE_JUMP_E + 0x07) # rel byte code offset, 16-bit unsigned + MP_BC_SETUP_EXCEPT = (MP_BC_BASE_JUMP_E + 0x08) # rel byte code offset, 16-bit unsigned + MP_BC_SETUP_FINALLY = (MP_BC_BASE_JUMP_E + 0x09) # rel byte code offset, 16-bit unsigned + MP_BC_POP_EXCEPT_JUMP = (MP_BC_BASE_JUMP_E + 0x0a) # rel byte code offset, 16-bit unsigned + MP_BC_FOR_ITER = (MP_BC_BASE_JUMP_E + 0x0b) # rel byte code offset, 16-bit unsigned + MP_BC_WITH_CLEANUP = (MP_BC_BASE_BYTE_O + 0x0c) + MP_BC_END_FINALLY = (MP_BC_BASE_BYTE_O + 0x0d) + MP_BC_GET_ITER = (MP_BC_BASE_BYTE_O + 0x0e) + MP_BC_GET_ITER_STACK = (MP_BC_BASE_BYTE_O + 0x0f) + + MP_BC_BUILD_TUPLE = (MP_BC_BASE_VINT_E + 0x0a) # uint + MP_BC_BUILD_LIST = (MP_BC_BASE_VINT_E + 0x0b) # uint + MP_BC_BUILD_MAP = (MP_BC_BASE_VINT_E + 0x0c) # uint + MP_BC_STORE_MAP = (MP_BC_BASE_BYTE_E + 0x02) + MP_BC_BUILD_SET = (MP_BC_BASE_VINT_E + 0x0d) # uint + MP_BC_BUILD_SLICE = (MP_BC_BASE_VINT_E + 0x0e) # uint + MP_BC_STORE_COMP = (MP_BC_BASE_VINT_E + 0x0f) # uint + MP_BC_UNPACK_SEQUENCE = (MP_BC_BASE_VINT_O + 0x00) # uint + MP_BC_UNPACK_EX = (MP_BC_BASE_VINT_O + 0x01) # uint + + MP_BC_RETURN_VALUE = (MP_BC_BASE_BYTE_E + 0x03) + MP_BC_RAISE_LAST = (MP_BC_BASE_BYTE_E + 0x04) + MP_BC_RAISE_OBJ = (MP_BC_BASE_BYTE_E + 0x05) + MP_BC_RAISE_FROM = (MP_BC_BASE_BYTE_E + 0x06) + MP_BC_YIELD_VALUE = (MP_BC_BASE_BYTE_E + 0x07) + MP_BC_YIELD_FROM = (MP_BC_BASE_BYTE_E + 0x08) + + MP_BC_MAKE_FUNCTION = (MP_BC_BASE_VINT_O + 0x02) # uint + MP_BC_MAKE_FUNCTION_DEFARGS = (MP_BC_BASE_VINT_O + 0x03) # uint + MP_BC_MAKE_CLOSURE = (MP_BC_BASE_VINT_E + 0x00) # uint; extra byte + MP_BC_MAKE_CLOSURE_DEFARGS = (MP_BC_BASE_VINT_E + 0x01) # uint; extra byte + MP_BC_CALL_FUNCTION = (MP_BC_BASE_VINT_O + 0x04) # uint + MP_BC_CALL_FUNCTION_VAR_KW = (MP_BC_BASE_VINT_O + 0x05) # uint + MP_BC_CALL_METHOD = (MP_BC_BASE_VINT_O + 0x06) # uint + MP_BC_CALL_METHOD_VAR_KW = (MP_BC_BASE_VINT_O + 0x07) # uint + + MP_BC_IMPORT_NAME = (MP_BC_BASE_QSTR_O + 0x0b) # qstr + MP_BC_IMPORT_FROM = (MP_BC_BASE_QSTR_O + 0x0c) # qstr + MP_BC_IMPORT_STAR = (MP_BC_BASE_BYTE_E + 0x09) + # fmt: on + + # Create a dict mapping opcode value to opcode name. + mapping = ["unknown" for _ in range(256)] + for op_name in list(locals()): + if op_name.startswith("MP_BC_"): + mapping[locals()[op_name]] = op_name[len("MP_BC_") :] + for i in range(MP_BC_LOAD_CONST_SMALL_INT_MULTI_NUM): + name = "LOAD_CONST_SMALL_INT %d" % (i - MP_BC_LOAD_CONST_SMALL_INT_MULTI_EXCESS) + mapping[MP_BC_LOAD_CONST_SMALL_INT_MULTI + i] = name + for i in range(MP_BC_LOAD_FAST_MULTI_NUM): + mapping[MP_BC_LOAD_FAST_MULTI + i] = "LOAD_FAST %d" % i + for i in range(MP_BC_STORE_FAST_MULTI_NUM): + mapping[MP_BC_STORE_FAST_MULTI + i] = "STORE_FAST %d" % i + for i in range(MP_BC_UNARY_OP_MULTI_NUM): + mapping[MP_BC_UNARY_OP_MULTI + i] = "UNARY_OP %d %s" % (i, mp_unary_op_method_name[i]) + for i in range(MP_BC_BINARY_OP_MULTI_NUM): + mapping[MP_BC_BINARY_OP_MULTI + i] = "BINARY_OP %d %s" % (i, mp_binary_op_method_name[i]) + # this function mirrors that in py/bc.c def mp_opcode_format(bytecode, ip, count_var_uint): @@ -147,6 +328,26 @@ def mp_opcode_format(bytecode, ip, count_var_uint): return f, ip - ip_start +def mp_opcode_decode(bytecode, ip): + opcode = bytecode[ip] + ip_start = ip + f = (0x000003A4 >> (2 * ((opcode) >> 4))) & 3 + extra_byte = (opcode & MP_BC_MASK_EXTRA_BYTE) == 0 + ip += 1 + arg = 0 + if f in (MP_BC_FORMAT_QSTR, MP_BC_FORMAT_VAR_UINT): + arg = bytecode[ip] & 0x7F + while bytecode[ip] & 0x80 != 0: + ip += 1 + arg = arg << 7 | bytecode[ip] & 0x7F + ip += 1 + elif f == MP_BC_FORMAT_OFFSET: + arg = bytecode[ip] | bytecode[ip + 1] << 8 + ip += 2 + ip += extra_byte + return f, ip - ip_start, arg + + def read_prelude_sig(read_byte): z = read_byte() # xSSSSEAA @@ -201,6 +402,7 @@ def extract_prelude(bytecode, ip): n_kwonly_args, n_def_pos_args, ) = read_prelude_sig(local_read_byte) + n_info, n_cell = read_prelude_size(local_read_byte) ip = ip_ref[0] @@ -208,74 +410,186 @@ def extract_prelude(bytecode, ip): ip = ip2 + n_info + n_cell # ip now points to first opcode # ip2 points to simple_name qstr - return ip, ip2, (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args) - -class MPFunTable: - pass + # Extract simple_name and argument qstrs (var uints). + args = [] + for arg_num in range(1 + n_pos_args + n_kwonly_args): + value = 0 + while True: + b = local_read_byte() + value = (value << 7) | (b & 0x7F) + if b & 0x80 == 0: + break + args.append(value) + + return ( + ip2, + ip, + ip_ref[0], + (n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args), + args, + ) -class RawCode(object): - # a set of all escaped names, to make sure they are unique - escaped_names = set() +class MPFunTable: + def __repr__(self): + return "mp_fun_table" - # convert code kind number to string - code_kind_str = { - MP_CODE_BYTECODE: "MP_CODE_BYTECODE", - MP_CODE_NATIVE_PY: "MP_CODE_NATIVE_PY", - MP_CODE_NATIVE_VIPER: "MP_CODE_NATIVE_VIPER", - MP_CODE_NATIVE_ASM: "MP_CODE_NATIVE_ASM", - } - def __init__(self, code_kind, bytecode, prelude_offset, qstrs, objs, raw_codes): - # set core variables - self.code_kind = code_kind - self.bytecode = bytecode - self.prelude_offset = prelude_offset - self.qstrs = qstrs - self.objs = objs - self.raw_codes = raw_codes - - if self.prelude_offset is None: - # no prelude, assign a dummy simple_name - self.prelude_offset = 0 - self.simple_name = global_qstrs[1] - else: - # extract prelude - self.ip, self.ip2, self.prelude = extract_prelude(self.bytecode, self.prelude_offset) - self.simple_name = self._unpack_qstr(self.ip2) - self.source_file = self._unpack_qstr(self.ip2 + 2) - self.line_info_offset = self.ip2 + 4 +class CompiledModule: + def __init__( + self, + mpy_source_file, + mpy_segments, + header, + qstr_table, + obj_table, + raw_code, + raw_code_file_offset, + escaped_name, + ): + self.mpy_source_file = mpy_source_file + self.mpy_segments = mpy_segments + self.source_file = qstr_table[0] + self.header = header + self.qstr_table = qstr_table + self.obj_table = obj_table + self.raw_code_file_offset = raw_code_file_offset + self.raw_code = raw_code + self.escaped_name = escaped_name def _unpack_qstr(self, ip): qst = self.bytecode[ip] | self.bytecode[ip + 1] << 8 return global_qstrs[qst] - def dump(self): - # dump children first - for rc in self.raw_codes: - rc.freeze("") - # TODO + def hexdump(self): + with open(self.mpy_source_file, "rb") as f: + WIDTH = 16 + COL_OFF = "\033[0m" + COL_TABLE = ( + ("", ""), # META + ("\033[0;31m", "\033[0;91m"), # QSTR + ("\033[0;32m", "\033[0;92m"), # OBJ + ("\033[0;34m", "\033[0;94m"), # CODE + ) + cur_col = "" + cur_col_index = 0 + offset = 0 + segment_index = 0 + while True: + data = bytes_cons(f.read(WIDTH)) + if not data: + break + + # Print out the hex dump of this line of data. + line_hex = cur_col + line_chr = cur_col + line_comment = "" + for i in range(len(data)): + # Determine the colour of the data, if any, and the line comment. + while segment_index < len(self.mpy_segments): + if offset + i == self.mpy_segments[segment_index].start: + cur_col = COL_TABLE[self.mpy_segments[segment_index].kind][ + cur_col_index + ] + cur_col_index = 1 - cur_col_index + line_hex += cur_col + line_chr += cur_col + line_comment += " %s%s%s" % ( + cur_col, + self.mpy_segments[segment_index].name, + COL_OFF, + ) + if offset + i == self.mpy_segments[segment_index].end: + cur_col = "" + line_hex += COL_OFF + line_chr += COL_OFF + segment_index += 1 + else: + break + + # Add to the hex part of the line. + if i % 2 == 0: + line_hex += " " + line_hex += "%02x" % data[i] + + # Add to the characters part of the line. + if 0x20 <= data[i] <= 0x7E: + line_chr += "%s" % chr(data[i]) + else: + line_chr += "." + + # Print out this line. + if cur_col: + line_hex += COL_OFF + line_chr += COL_OFF + pad = " " * ((WIDTH - len(data)) * 5 // 2) + print("%08x:%s%s %s %s" % (offset, line_hex, pad, line_chr, line_comment)) + offset += WIDTH + + def disassemble(self): + print("mpy_source_file:", self.mpy_source_file) + print("source_file:", self.source_file.str) + print("header:", hexlify_to_str(self.header)) + print("qstr_table[%u]:" % len(self.qstr_table)) + for q in self.qstr_table: + print(" %s" % q.str) + print("obj_table:", self.obj_table) + self.raw_code.disassemble() + + def freeze(self, compiled_module_index): + print() + print("/" * 80) + print("// frozen module %s" % self.escaped_name) + print("// - original source file: %s" % self.mpy_source_file) + print("// - frozen file name: %s" % self.source_file.str) + print("// - .mpy header: %s" % ":".join("%02x" % b for b in self.header)) + print() - def freeze_children(self, parent_name): - self.escaped_name = parent_name + self.simple_name.qstr_esc + self.raw_code.freeze() + print() - # make sure the escaped name is unique - i = 2 - while self.escaped_name in RawCode.escaped_names: - self.escaped_name = parent_name + self.simple_name.qstr_esc + str(i) - i += 1 - RawCode.escaped_names.add(self.escaped_name) + self.freeze_constants() - # emit children first - for rc in self.raw_codes: - rc.freeze(self.escaped_name + "_") + print() + print("static const mp_frozen_module_t frozen_module_%s = {" % self.escaped_name) + print(" .constants = {") + if len(self.qstr_table): + print( + " .qstr_table = (qstr_short_t *)&const_qstr_table_data_%s," + % self.escaped_name + ) + else: + print(" .qstr_table = NULL,") + if len(self.obj_table): + print(" .obj_table = (mp_obj_t *)&const_obj_table_data_%s," % self.escaped_name) + else: + print(" .obj_table = NULL,") + print(" },") + print(" .rc = &raw_code_%s," % self.raw_code.escaped_name) + print("};") def freeze_constants(self): + global const_str_content, const_int_content, const_obj_content + + if len(self.qstr_table): + print( + "static const qstr_short_t const_qstr_table_data_%s[%u] = {" + % (self.escaped_name, len(self.qstr_table)) + ) + for q in self.qstr_table: + print(" %s," % q.qstr_id) + print("};") + + if not len(self.obj_table): + return + # generate constant objects - for i, obj in enumerate(self.objs): + print() + print("// constants") + for i, obj in enumerate(self.obj_table): obj_name = "const_obj_%s_%u" % (self.escaped_name, i) - if obj is MPFunTable: + if isinstance(obj, MPFunTable): pass elif obj is Ellipsis: print("#define %s mp_const_ellipsis_obj" % obj_name) @@ -286,7 +600,7 @@ class RawCode(object): else: obj_type = "mp_type_bytes" print( - 'STATIC const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};' + 'static const mp_obj_str_t %s = {{&%s}, %u, %u, (const byte*)"%s"};' % ( obj_name, obj_type, @@ -295,6 +609,8 @@ class RawCode(object): "".join(("\\x%02x" % b) for b in obj), ) ) + const_str_content += len(obj) + const_obj_content += 4 * 4 elif is_int_type(obj): if config.MICROPY_LONGINT_IMPL == config.MICROPY_LONGINT_IMPL_NONE: # TODO check if we can actually fit this long-int into a small-int @@ -316,76 +632,139 @@ class RawCode(object): ndigs = len(digs) digs = ",".join(("%#x" % d) for d in digs) print( - "STATIC const mp_obj_int_t %s = {{&mp_type_int}, " + "static const mp_obj_int_t %s = {{&mp_type_int}, " "{.neg=%u, .fixed_dig=1, .alloc=%u, .len=%u, .dig=(uint%u_t*)(const uint%u_t[]){%s}}};" % (obj_name, neg, ndigs, ndigs, bits_per_dig, bits_per_dig, digs) ) + const_int_content += (digs.count(",") + 1) * bits_per_dig // 8 + const_obj_content += 4 * 4 elif type(obj) is float: print( "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" ) print( - "STATIC const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};" + "static const mp_obj_float_t %s = {{&mp_type_float}, (mp_float_t)%.16g};" % (obj_name, obj) ) print("#endif") + const_obj_content += 3 * 4 elif type(obj) is complex: print( - "STATIC const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};" + "static const mp_obj_complex_t %s = {{&mp_type_complex}, (mp_float_t)%.16g, (mp_float_t)%.16g};" % (obj_name, obj.real, obj.imag) ) else: raise FreezeError(self, "freezing of object %r is not implemented" % (obj,)) - # generate constant table, if it has any entries - const_table_len = len(self.qstrs) + len(self.objs) + len(self.raw_codes) - if const_table_len: - print( - "STATIC const mp_rom_obj_t const_table_data_%s[%u] = {" - % (self.escaped_name, const_table_len) - ) - for qst in self.qstrs: - print(" MP_ROM_QSTR(%s)," % global_qstrs[qst].qstr_id) - for i in range(len(self.objs)): - if self.objs[i] is MPFunTable: - print(" &mp_fun_table,") - elif type(self.objs[i]) is float: - print( - "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" - ) - print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) - print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C") - n = struct.unpack("<I", struct.pack("<f", self.objs[i]))[0] - n = ((n & ~0x3) | 2) + 0x80800000 - print(" (mp_rom_obj_t)(0x%08x)," % (n,)) - print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D") - n = struct.unpack("<Q", struct.pack("<d", self.objs[i]))[0] - n += 0x8004000000000000 - print(" (mp_rom_obj_t)(0x%016x)," % (n,)) - print("#endif") - else: - print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) - for rc in self.raw_codes: - print(" MP_ROM_PTR(&raw_code_%s)," % rc.escaped_name) + # generate constant table + print() + print("// constant table") + print( + "static const mp_rom_obj_t const_obj_table_data_%s[%u] = {" + % (self.escaped_name, len(self.obj_table)) + ) + for i in range(len(self.obj_table)): + if isinstance(self.obj_table[i], MPFunTable): + print(" &mp_fun_table,") + elif type(self.obj_table[i]) is float: + print( + "#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_A || MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_B" + ) + print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) + print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_C") + n = struct.unpack("<I", struct.pack("<f", self.obj_table[i]))[0] + n = ((n & ~0x3) | 2) + 0x80800000 + print(" (mp_rom_obj_t)(0x%08x)," % (n,)) + print("#elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D") + n = struct.unpack("<Q", struct.pack("<d", self.obj_table[i]))[0] + n += 0x8004000000000000 + print(" (mp_rom_obj_t)(0x%016x)," % (n,)) + print("#endif") + else: + print(" MP_ROM_PTR(&const_obj_%s_%u)," % (self.escaped_name, i)) + print("};") + + global const_table_ptr_content + const_table_ptr_content += len(self.obj_table) + + +class RawCode(object): + # a set of all escaped names, to make sure they are unique + escaped_names = set() + + # convert code kind number to string + code_kind_str = { + MP_CODE_BYTECODE: "MP_CODE_BYTECODE", + MP_CODE_NATIVE_PY: "MP_CODE_NATIVE_PY", + MP_CODE_NATIVE_VIPER: "MP_CODE_NATIVE_VIPER", + MP_CODE_NATIVE_ASM: "MP_CODE_NATIVE_ASM", + } + + def __init__(self, cm_escaped_name, qstr_table, fun_data, prelude_offset, code_kind): + self.qstr_table = qstr_table + self.fun_data = fun_data + self.prelude_offset = prelude_offset + self.code_kind = code_kind + + if code_kind in (MP_CODE_BYTECODE, MP_CODE_NATIVE_PY): + ( + self.offset_names, + self.offset_opcodes, + self.offset_line_info, + self.prelude, + self.names, + ) = extract_prelude(self.fun_data, prelude_offset) + self.scope_flags = self.prelude[2] + self.n_pos_args = self.prelude[3] + self.simple_name = self.qstr_table[self.names[0]] + else: + self.simple_name = self.qstr_table[0] + + escaped_name = cm_escaped_name + "_" + self.simple_name.qstr_esc + + # make sure the escaped name is unique + i = 2 + unique_escaped_name = escaped_name + while unique_escaped_name in self.escaped_names: + unique_escaped_name = escaped_name + str(i) + i += 1 + self.escaped_names.add(unique_escaped_name) + self.escaped_name = unique_escaped_name + + def disassemble_children(self): + print(" children:", [rc.simple_name.str for rc in self.children]) + for rc in self.children: + rc.disassemble() + + def freeze_children(self): + # Freeze children and generate table of children. + if len(self.children): + for rc in self.children: + print("// child of %s" % self.escaped_name) + rc.freeze() + print() + print("static const mp_raw_code_t *const children_%s[] = {" % self.escaped_name) + for rc in self.children: + print(" &raw_code_%s," % rc.escaped_name) print("};") + print() - def freeze_module(self, qstr_links=(), type_sig=0): - # generate module - if self.simple_name.str != "<module>": - print("STATIC ", end="") - print("const mp_raw_code_t raw_code_%s = {" % self.escaped_name) + def freeze_raw_code(self, qstr_links=(), type_sig=0): + # Generate mp_raw_code_t. + print("static const mp_raw_code_t raw_code_%s = {" % self.escaped_name) print(" .kind = %s," % RawCode.code_kind_str[self.code_kind]) - print(" .scope_flags = 0x%02x," % self.prelude[2]) - print(" .n_pos_args = %u," % self.prelude[3]) + print(" .scope_flags = 0x%02x," % self.scope_flags) + print(" .n_pos_args = %u," % self.n_pos_args) print(" .fun_data = fun_data_%s," % self.escaped_name) - if len(self.qstrs) + len(self.objs) + len(self.raw_codes): - print(" .const_table = (mp_uint_t*)const_table_data_%s," % self.escaped_name) + print(" #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS") + print(" .fun_data_len = %u," % len(self.fun_data)) + print(" #endif") + if len(self.children): + print(" .children = (void *)&children_%s," % self.escaped_name) else: - print(" .const_table = NULL,") + print(" .children = NULL,") print(" #if MICROPY_PERSISTENT_CODE_SAVE") - print(" .fun_data_len = %u," % len(self.bytecode)) - print(" .n_obj = %u," % len(self.objs)) - print(" .n_raw_code = %u," % len(self.raw_codes)) + print(" .n_children = %u," % len(self.children)) if self.code_kind == MP_CODE_BYTECODE: print(" #if MICROPY_PY_SYS_SETTRACE") print(" .prelude = {") @@ -395,13 +774,14 @@ class RawCode(object): print(" .n_pos_args = %u," % self.prelude[3]) print(" .n_kwonly_args = %u," % self.prelude[4]) print(" .n_def_pos_args = %u," % self.prelude[5]) - print(" .qstr_block_name = %s," % self.simple_name.qstr_id) - print(" .qstr_source_file = %s," % self.source_file.qstr_id) + print(" .qstr_block_name_idx = %u," % self.names[0]) print( " .line_info = fun_data_%s + %u," - % (self.escaped_name, self.line_info_offset) + % (self.escaped_name, self.offset_line_info) + ) + print( + " .opcodes = fun_data_%s + %u," % (self.escaped_name, self.offset_opcodes) ) - print(" .opcodes = fun_data_%s + %u," % (self.escaped_name, self.ip)) print(" },") print(" .line_of_definition = %u," % 0) # TODO print(" #endif") @@ -416,66 +796,109 @@ class RawCode(object): print(" #endif") print("};") + global raw_code_count, raw_code_content + raw_code_count += 1 + raw_code_content += 4 * 4 + class RawCodeBytecode(RawCode): - def __init__(self, bytecode, qstrs, objs, raw_codes): + def __init__(self, cm_escaped_name, qstr_table, obj_table, fun_data): + self.obj_table = obj_table super(RawCodeBytecode, self).__init__( - MP_CODE_BYTECODE, bytecode, 0, qstrs, objs, raw_codes + cm_escaped_name, qstr_table, fun_data, 0, MP_CODE_BYTECODE ) - def freeze(self, parent_name): - self.freeze_children(parent_name) + def disassemble(self): + bc = self.fun_data + print("simple_name:", self.simple_name.str) + print(" raw bytecode:", len(bc), hexlify_to_str(bc)) + print(" prelude:", self.prelude) + print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) + print(" line info:", hexlify_to_str(bc[self.offset_line_info : self.offset_opcodes])) + ip = self.offset_opcodes + while ip < len(bc): + fmt, sz, arg = mp_opcode_decode(bc, ip) + if bc[ip] == Opcodes.MP_BC_LOAD_CONST_OBJ: + arg = "%r" % self.obj_table[arg] + if fmt == MP_BC_FORMAT_QSTR: + arg = self.qstr_table[arg].str + elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET): + pass + else: + arg = "" + print( + " %-11s %s %s" % (hexlify_to_str(bc[ip : ip + sz]), Opcodes.mapping[bc[ip]], arg) + ) + ip += sz + self.disassemble_children() + def freeze(self): # generate bytecode data - print() + bc = self.fun_data print( - "// frozen bytecode for file %s, scope %s%s" - % (self.source_file.str, parent_name, self.simple_name.str) + "// frozen bytecode for file %s, scope %s" + % (self.qstr_table[0].str, self.escaped_name) ) - print("STATIC const byte fun_data_%s[%u] = {" % (self.escaped_name, len(self.bytecode))) - print(" ", end="") - for i in range(self.ip2): - print(" 0x%02x," % self.bytecode[i], end="") - print() - print(" ", self.simple_name.qstr_id, "& 0xff,", self.simple_name.qstr_id, ">> 8,") - print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") - print(" ", end="") - for i in range(self.ip2 + 4, self.ip): - print(" 0x%02x," % self.bytecode[i], end="") - print() - ip = self.ip - while ip < len(self.bytecode): - f, sz = mp_opcode_format(self.bytecode, ip, True) - if f == 1: - qst = self._unpack_qstr(ip + 1).qstr_id - extra = "" if sz == 3 else " 0x%02x," % self.bytecode[ip + 3] - print(" ", "0x%02x," % self.bytecode[ip], qst, "& 0xff,", qst, ">> 8,", extra) - else: - print(" ", "".join("0x%02x, " % self.bytecode[ip + i] for i in range(sz))) + print("static const byte fun_data_%s[%u] = {" % (self.escaped_name, len(bc))) + + print(" ", end="") + for b in bc[: self.offset_names]: + print("0x%02x," % b, end="") + print(" // prelude") + + print(" ", end="") + for b in bc[self.offset_names : self.offset_line_info]: + print("0x%02x," % b, end="") + print(" // names: %s" % ", ".join(self.qstr_table[i].str for i in self.names)) + + print(" ", end="") + for b in bc[self.offset_line_info : self.offset_opcodes]: + print("0x%02x," % b, end="") + print(" // code info") + + ip = self.offset_opcodes + while ip < len(bc): + fmt, sz, arg = mp_opcode_decode(bc, ip) + opcode_name = Opcodes.mapping[bc[ip]] + if fmt == MP_BC_FORMAT_QSTR: + opcode_name += " " + self.qstr_table[arg].str + elif fmt in (MP_BC_FORMAT_VAR_UINT, MP_BC_FORMAT_OFFSET): + opcode_name += " %u" % arg + print( + " %s, // %s" % (",".join("0x%02x" % b for b in bc[ip : ip + sz]), opcode_name) + ) ip += sz + print("};") - self.freeze_constants() - self.freeze_module() + self.freeze_children() + self.freeze_raw_code() + + global bc_content + bc_content += len(bc) class RawCodeNative(RawCode): def __init__( self, - code_kind, + cm_escaped_name, + qstr_table, + kind, fun_data, prelude_offset, - prelude, qstr_links, - qstrs, - objs, - raw_codes, + scope_flags, + n_pos_args, type_sig, ): super(RawCodeNative, self).__init__( - code_kind, fun_data, prelude_offset, qstrs, objs, raw_codes + cm_escaped_name, qstr_table, fun_data, prelude_offset, kind ) - self.prelude = prelude + + if kind in (MP_CODE_NATIVE_VIPER, MP_CODE_NATIVE_ASM): + self.scope_flags = scope_flags + self.n_pos_args = n_pos_args + self.qstr_links = qstr_links self.type_sig = type_sig if config.native_arch in ( @@ -502,11 +925,32 @@ class RawCodeNative(RawCode): # ARMVxxM -- two byte align. self.fun_data_attributes += " __attribute__ ((aligned (2)))" + def disassemble(self): + fun_data = self.fun_data + print("simple_name:", self.simple_name.str) + print( + " raw data:", + len(fun_data), + hexlify_to_str(fun_data[:32]), + "..." if len(fun_data) > 32 else "", + ) + if self.code_kind != MP_CODE_NATIVE_PY: + return + print(" prelude:", self.prelude) + print(" args:", [self.qstr_table[i].str for i in self.names[1:]]) + print(" line info:", fun_data[self.offset_line_info : self.offset_opcodes]) + ip = 0 + while ip < self.prelude_offset: + sz = 16 + print(" ", hexlify_to_str(fun_data[ip : min(ip + sz, self.prelude_offset)])) + ip += sz + self.disassemble_children() + def _asm_thumb_rewrite_mov(self, pc, val): - print(" (%u & 0xf0) | (%s >> 12)," % (self.bytecode[pc], val), end="") - print(" (%u & 0xfb) | (%s >> 9 & 0x04)," % (self.bytecode[pc + 1], val), end="") + print(" (%u & 0xf0) | (%s >> 12)," % (self.fun_data[pc], val), end="") + print(" (%u & 0xfb) | (%s >> 9 & 0x04)," % (self.fun_data[pc + 1], val), end="") print(" (%s & 0xff)," % (val,), end="") - print(" (%u & 0x07) | (%s >> 4 & 0x70)," % (self.bytecode[pc + 3], val)) + print(" (%u & 0x07) | (%s >> 4 & 0x70)," % (self.fun_data[pc + 3], val)) def _link_qstr(self, pc, kind, qst): if kind == 0: @@ -543,32 +987,22 @@ class RawCodeNative(RawCode): else: assert 0 - def freeze(self, parent_name): - if self.prelude[2] & ~0x0F: + def freeze(self): + if self.scope_flags & ~0x0F: raise FreezeError("unable to freeze code with relocations") - self.freeze_children(parent_name) - # generate native code data print() - if self.code_kind == MP_CODE_NATIVE_PY: - print( - "// frozen native code for file %s, scope %s%s" - % (self.source_file.str, parent_name, self.simple_name.str) - ) - elif self.code_kind == MP_CODE_NATIVE_VIPER: - print("// frozen viper code for scope %s" % (parent_name,)) - else: - print("// frozen assembler code for scope %s" % (parent_name,)) print( - "STATIC const byte fun_data_%s[%u] %s = {" - % (self.escaped_name, len(self.bytecode), self.fun_data_attributes) + "// frozen native code for file %s, scope %s" + % (self.qstr_table[0].str, self.escaped_name) + ) + print( + "static const byte fun_data_%s[%u] %s = {" + % (self.escaped_name, len(self.fun_data), self.fun_data_attributes) ) - if self.code_kind == MP_CODE_NATIVE_PY: - i_top = self.prelude_offset - else: - i_top = len(self.bytecode) + i_top = len(self.fun_data) i = 0 qi = 0 while i < i_top: @@ -585,229 +1019,253 @@ class RawCodeNative(RawCode): i16 = min(i16, self.qstr_links[qi][0]) print(" ", end="") for ii in range(i, i16): - print(" 0x%02x," % self.bytecode[ii], end="") + print(" 0x%02x," % self.fun_data[ii], end="") print() i = i16 - if self.code_kind == MP_CODE_NATIVE_PY: - print(" ", end="") - for i in range(self.prelude_offset, self.ip2): - print(" 0x%02x," % self.bytecode[i], end="") - print() - - print(" ", self.simple_name.qstr_id, "& 0xff,", self.simple_name.qstr_id, ">> 8,") - print(" ", self.source_file.qstr_id, "& 0xff,", self.source_file.qstr_id, ">> 8,") - - print(" ", end="") - for i in range(self.ip2 + 4, self.ip): - print(" 0x%02x," % self.bytecode[i], end="") - print() - print("};") - self.freeze_constants() - self.freeze_module(self.qstr_links, self.type_sig) + self.freeze_children() + self.freeze_raw_code(self.qstr_links, self.type_sig) -class BytecodeBuffer: - def __init__(self, size): - self.buf = bytearray(size) - self.idx = 0 +class MPYSegment: + META = 0 + QSTR = 1 + OBJ = 2 + CODE = 3 - def is_full(self): - return self.idx == len(self.buf) + def __init__(self, kind, name, start, end): + self.kind = kind + self.name = name + self.start = start + self.end = end - def append(self, b): - self.buf[self.idx] = b - self.idx += 1 +class MPYReader: + def __init__(self, filename, fileobj): + self.filename = filename + self.fileobj = fileobj -def read_byte(f, out=None): - b = bytes_cons(f.read(1))[0] - if out is not None: - out.append(b) - return b + def tell(self): + return self.fileobj.tell() + def read_byte(self): + return bytes_cons(self.fileobj.read(1))[0] -def read_uint(f, out=None): - i = 0 - while True: - b = read_byte(f, out) - i = (i << 7) | (b & 0x7F) - if b & 0x80 == 0: - break - return i + def read_bytes(self, n): + return bytes_cons(self.fileobj.read(n)) + def read_uint(self): + i = 0 + while True: + b = self.read_byte() + i = (i << 7) | (b & 0x7F) + if b & 0x80 == 0: + break + return i -def read_qstr(f, qstr_win): - ln = read_uint(f) - if ln == 0: - # static qstr - return bytes_cons(f.read(1))[0] + +def read_qstr(reader, segments): + start_pos = reader.tell() + ln = reader.read_uint() if ln & 1: - # qstr in table - return qstr_win.access(ln >> 1) + # static qstr + segments.append( + MPYSegment(MPYSegment.META, global_qstrs[ln >> 1].str, start_pos, start_pos) + ) + return ln >> 1 ln >>= 1 - data = str_cons(f.read(ln), "utf8") + start_pos = reader.tell() + data = str_cons(reader.read_bytes(ln), "utf8") + reader.read_byte() # read and discard null terminator + segments.append(MPYSegment(MPYSegment.QSTR, data, start_pos, reader.tell())) global_qstrs.append(QStrType(data)) - qstr_win.push(len(global_qstrs) - 1) return len(global_qstrs) - 1 -def read_obj(f): - obj_type = f.read(1) - if obj_type == b"e": +def read_obj(reader, segments): + obj_type = reader.read_bytes(1) + if obj_type == b"t": + return MPFunTable() + elif obj_type == b"e": return Ellipsis else: - buf = f.read(read_uint(f)) + ln = reader.read_uint() + start_pos = reader.tell() + buf = reader.read_bytes(ln) + if obj_type in (b"s", b"b"): + reader.read_byte() # read and discard null terminator if obj_type == b"s": - return str_cons(buf, "utf8") + obj = str_cons(buf, "utf8") elif obj_type == b"b": - return bytes_cons(buf) + obj = buf elif obj_type == b"i": - return int(str_cons(buf, "ascii"), 10) + obj = int(str_cons(buf, "ascii"), 10) elif obj_type == b"f": - return float(str_cons(buf, "ascii")) + obj = float(str_cons(buf, "ascii")) elif obj_type == b"c": - return complex(str_cons(buf, "ascii")) + obj = complex(str_cons(buf, "ascii")) else: - assert 0 + raise MPYReadError(reader.filename, "corrupt .mpy file") + segments.append(MPYSegment(MPYSegment.OBJ, obj, start_pos, reader.tell())) + return obj -def read_prelude(f, bytecode, qstr_win): - ( - n_state, - n_exc_stack, - scope_flags, - n_pos_args, - n_kwonly_args, - n_def_pos_args, - ) = read_prelude_sig(lambda: read_byte(f, bytecode)) - n_info, n_cell = read_prelude_size(lambda: read_byte(f, bytecode)) - read_qstr_and_pack(f, bytecode, qstr_win) # simple_name - read_qstr_and_pack(f, bytecode, qstr_win) # source_file - for _ in range(n_info - 4 + n_cell): - read_byte(f, bytecode) - return n_state, n_exc_stack, scope_flags, n_pos_args, n_kwonly_args, n_def_pos_args - - -def read_qstr_and_pack(f, bytecode, qstr_win): - qst = read_qstr(f, qstr_win) - bytecode.append(qst & 0xFF) - bytecode.append(qst >> 8) - - -def read_bytecode(file, bytecode, qstr_win): - while not bytecode.is_full(): - op = read_byte(file, bytecode) - f, sz = mp_opcode_format(bytecode.buf, bytecode.idx - 1, False) - sz -= 1 - if f == MP_BC_FORMAT_QSTR: - read_qstr_and_pack(file, bytecode, qstr_win) - sz -= 2 - elif f == MP_BC_FORMAT_VAR_UINT: - while read_byte(file, bytecode) & 0x80: - pass - for _ in range(sz): - read_byte(file, bytecode) - - -def read_raw_code(f, qstr_win): - kind_len = read_uint(f) +def read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments): + # Read raw code header. + kind_len = reader.read_uint() kind = (kind_len & 3) + MP_CODE_BYTECODE - fun_data_len = kind_len >> 2 - fun_data = BytecodeBuffer(fun_data_len) + has_children = (kind_len >> 2) & 1 + fun_data_len = kind_len >> 3 + + # Read the body of the raw code. + file_offset = reader.tell() + fun_data = reader.read_bytes(fun_data_len) + segments_len = len(segments) if kind == MP_CODE_BYTECODE: - prelude = read_prelude(f, fun_data, qstr_win) - read_bytecode(f, fun_data, qstr_win) + # Create bytecode raw code. + rc = RawCodeBytecode(cm_escaped_name, qstr_table, obj_table, fun_data) else: - fun_data.buf[:] = f.read(fun_data_len) - + # Create native raw code. qstr_links = [] if kind in (MP_CODE_NATIVE_PY, MP_CODE_NATIVE_VIPER): - # load qstr link table - n_qstr_link = read_uint(f) + # Read qstr link table. + n_qstr_link = reader.read_uint() for _ in range(n_qstr_link): - off = read_uint(f) - qst = read_qstr(f, qstr_win) + off = reader.read_uint() + qst = read_qstr(reader, segments) qstr_links.append((off >> 2, off & 3, qst)) - type_sig = 0 + native_scope_flags = 0 + native_n_pos_args = 0 + native_type_sig = 0 if kind == MP_CODE_NATIVE_PY: - prelude_offset = read_uint(f) - _, name_idx, prelude = extract_prelude(fun_data.buf, prelude_offset) - fun_data.idx = name_idx # rewind to where qstrs are in prelude - read_qstr_and_pack(f, fun_data, qstr_win) # simple_name - read_qstr_and_pack(f, fun_data, qstr_win) # source_file + prelude_offset = reader.read_uint() else: - prelude_offset = None - scope_flags = read_uint(f) - n_pos_args = 0 - if kind == MP_CODE_NATIVE_ASM: - n_pos_args = read_uint(f) - type_sig = read_uint(f) - prelude = (None, None, scope_flags, n_pos_args, 0) - - qstrs = [] - objs = [] - raw_codes = [] - if kind != MP_CODE_NATIVE_ASM: - # load constant table - n_obj = read_uint(f) - n_raw_code = read_uint(f) - qstrs = [read_qstr(f, qstr_win) for _ in range(prelude[3] + prelude[4])] - if kind != MP_CODE_BYTECODE: - objs.append(MPFunTable) - objs.extend([read_obj(f) for _ in range(n_obj)]) - raw_codes = [read_raw_code(f, qstr_win) for _ in range(n_raw_code)] + prelude_offset = 0 + native_scope_flags = reader.read_uint() + if kind == MP_CODE_NATIVE_VIPER: + # Read any additional sections for native viper. + if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: + rodata_size = reader.read_uint() + if native_scope_flags & MP_SCOPE_FLAG_VIPERBSS: + bss_size = reader.read_uint() + if native_scope_flags & MP_SCOPE_FLAG_VIPERRODATA: + reader.read_bytes(rodata_size) + if native_scope_flags & MP_SCOPE_FLAG_VIPERRELOC: + while True: + op = reader.read_byte() + if op == 0xFF: + break + if op & 1: + addr = reader.read_uint() + op >>= 1 + if op <= 5 and op & 1: + n = reader.read_uint() + else: + assert kind == MP_CODE_NATIVE_ASM + native_n_pos_args = reader.read_uint() + native_type_sig = reader.read_uint() - if kind == MP_CODE_BYTECODE: - return RawCodeBytecode(fun_data.buf, qstrs, objs, raw_codes) - else: - return RawCodeNative( + rc = RawCodeNative( + cm_escaped_name, + qstr_table, kind, - fun_data.buf, + fun_data, prelude_offset, - prelude, qstr_links, - qstrs, - objs, - raw_codes, - type_sig, + native_scope_flags, + native_n_pos_args, + native_type_sig, ) + # Add a segment for the raw code data. + segments.insert( + segments_len, + MPYSegment(MPYSegment.CODE, rc.simple_name.str, file_offset, file_offset + fun_data_len), + ) + + # Read children, if there are any. + rc.children = [] + if has_children: + n_children = reader.read_uint() + for _ in range(n_children): + rc.children.append( + read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments) + ) + + return rc + def read_mpy(filename): - with open(filename, "rb") as f: - header = bytes_cons(f.read(4)) + with open(filename, "rb") as fileobj: + reader = MPYReader(filename, fileobj) + segments = [] + + # Read and verify the header. + header = reader.read_bytes(4) if header[0] != ord("M"): - raise Exception("not a valid .mpy file") + raise MPYReadError(filename, "not a valid .mpy file") if header[1] != config.MPY_VERSION: - raise Exception("incompatible .mpy version") + raise MPYReadError(filename, "incompatible .mpy version") feature_byte = header[2] - qw_size = read_uint(f) config.MICROPY_PY_BUILTINS_STR_UNICODE = (feature_byte & 2) != 0 mpy_native_arch = feature_byte >> 2 if mpy_native_arch != MP_NATIVE_ARCH_NONE: if config.native_arch == MP_NATIVE_ARCH_NONE: config.native_arch = mpy_native_arch elif config.native_arch != mpy_native_arch: - raise Exception("native architecture mismatch") + raise MPYReadError(filename, "native architecture mismatch") config.mp_small_int_bits = header[3] - qstr_win = QStrWindow(qw_size) - rc = read_raw_code(f, qstr_win) - rc.mpy_source_file = filename - rc.qstr_win_size = qw_size - return rc + + # Read number of qstrs, and number of objects. + n_qstr = reader.read_uint() + n_obj = reader.read_uint() + + # Read qstrs and construct qstr table. + qstr_table = [] + for i in range(n_qstr): + q = read_qstr(reader, segments) + qstr_table.append(global_qstrs[q]) + + # Read objects and construct object table. + obj_table = [] + for i in range(n_obj): + obj_table.append(read_obj(reader, segments)) + + # Compute the compiled-module escaped name. + cm_escaped_name = qstr_table[0].str.replace("/", "_")[:-3] + + # Read the outer raw code, which will in turn read all its children. + raw_code_file_offset = reader.tell() + raw_code = read_raw_code(reader, cm_escaped_name, qstr_table, obj_table, segments) + + # Create the outer-level compiled module representing the whole .mpy file. + return CompiledModule( + filename, + segments, + header, + qstr_table, + obj_table, + raw_code, + raw_code_file_offset, + cm_escaped_name, + ) -def dump_mpy(raw_codes): - for rc in raw_codes: - rc.dump() +def hexdump_mpy(compiled_modules): + for cm in compiled_modules: + cm.hexdump() -def freeze_mpy(base_qstrs, raw_codes): +def disassemble_mpy(compiled_modules): + for cm in compiled_modules: + cm.disassemble() + + +def freeze_mpy(base_qstrs, compiled_modules): # add to qstrs new = {} for q in global_qstrs: @@ -864,6 +1322,17 @@ def freeze_mpy(base_qstrs, raw_codes): # As in qstr.c, set so that the first dynamically allocated pool is twice this size; must be <= the len qstr_pool_alloc = min(len(new), 10) + global bc_content, const_str_content, const_int_content, const_obj_content, const_table_qstr_content, const_table_ptr_content, raw_code_count, raw_code_content + qstr_content = 0 + bc_content = 0 + const_str_content = 0 + const_int_content = 0 + const_obj_content = 0 + const_table_qstr_content = 0 + const_table_ptr_content = 0 + raw_code_count = 0 + raw_code_content = 0 + print() print("const qstr_hash_t mp_qstr_frozen_const_hashes[] = {") qstr_size = {"metadata": 0, "data": 0} @@ -892,32 +1361,49 @@ def freeze_mpy(base_qstrs, raw_codes): print(" {") for _, _, qstr, qbytes in new: print(' "%s",' % qstrutil.escape_bytes(qstr, qbytes)) + qstr_content += ( + config.MICROPY_QSTR_BYTES_IN_LEN + config.MICROPY_QSTR_BYTES_IN_HASH + len(qbytes) + 1 + ) print(" },") print("};") - for rc in raw_codes: - rc.freeze(rc.source_file.str.replace("/", "_")[:-3] + "_") + # Freeze all modules. + for idx, cm in enumerate(compiled_modules): + cm.freeze(idx) + + # Print separator, separating individual modules from global data structures. + print() + print("/" * 80) + print("// collection of all frozen modules") + # Define the string of frozen module names. print() print("const char mp_frozen_names[] = {") - print("#ifdef MP_FROZEN_STR_NAMES") + print(" #ifdef MP_FROZEN_STR_NAMES") # makemanifest.py might also include some frozen string content. - print("MP_FROZEN_STR_NAMES") - print("#endif") - for rc in raw_codes: - module_name = rc.source_file.str - print('"%s\\0"' % module_name) - print('"\\0"};') - - print("const mp_raw_code_t *const mp_frozen_mpy_content[] = {") - for rc in raw_codes: - print(" &raw_code_%s," % rc.escaped_name) + print(" MP_FROZEN_STR_NAMES") + print(" #endif") + mp_frozen_mpy_names_content = 1 + for cm in compiled_modules: + module_name = cm.source_file.str + print(' "%s\\0"' % module_name) + mp_frozen_mpy_names_content += len(cm.source_file.str) + 1 + print(' "\\0"') + print("};") + + # Define the array of pointers to frozen module content. + print() + print("const mp_frozen_module_t *const mp_frozen_mpy_content[] = {") + for cm in compiled_modules: + print(" &frozen_module_%s," % cm.escaped_name) print("};") + mp_frozen_mpy_content_size = len(compiled_modules * 4) # If a port defines MICROPY_FROZEN_LIST_ITEM then list all modules wrapped in that macro. + print() print("#ifdef MICROPY_FROZEN_LIST_ITEM") - for rc in raw_codes: - module_name = rc.source_file.str + for cm in compiled_modules: + module_name = cm.source_file.str if module_name.endswith("/__init__.py"): short_name = module_name[: -len("/__init__.py")] else: @@ -925,45 +1411,94 @@ def freeze_mpy(base_qstrs, raw_codes): print('MICROPY_FROZEN_LIST_ITEM("%s", "%s")' % (short_name, module_name)) print("#endif") + print() + print("/*") + print("byte sizes:") + print("qstr content: %d unique, %d bytes" % (len(new), qstr_content)) + print("bc content: %d" % bc_content) + print("const str content: %d" % const_str_content) + print("const int content: %d" % const_int_content) + print("const obj content: %d" % const_obj_content) + print( + "const table qstr content: %d entries, %d bytes" + % (const_table_qstr_content, const_table_qstr_content * 4) + ) + print( + "const table ptr content: %d entries, %d bytes" + % (const_table_ptr_content, const_table_ptr_content * 4) + ) + print("raw code content: %d * 4 = %d" % (raw_code_count, raw_code_content)) + print("mp_frozen_mpy_names_content: %d" % mp_frozen_mpy_names_content) + print("mp_frozen_mpy_content_size: %d" % mp_frozen_mpy_content_size) + print( + "total: %d" + % ( + qstr_content + + bc_content + + const_str_content + + const_int_content + + const_obj_content + + const_table_qstr_content * 4 + + const_table_ptr_content * 4 + + raw_code_content + + mp_frozen_mpy_names_content + + mp_frozen_mpy_content_size + ) + ) + print("*/") + def merge_mpy(raw_codes, output_file): - assert len(raw_codes) <= 31 # so var-uints all fit in 1 byte + assert len(raw_codes) <= 2 # so var-uints all fit in 1 byte merged_mpy = bytearray() if len(raw_codes) == 1: with open(raw_codes[0].mpy_source_file, "rb") as f: merged_mpy.extend(f.read()) else: - header = bytearray(5) + main_rc = None + for rc in raw_codes: + if len(rc.qstr_table) > 1 or len(rc.obj_table) > 0: + # Must use qstr_table and obj_table from this raw_code + if main_rc is not None: + raise Exception( + "can't merge files when more than one has a populated qstr or obj table" + ) + main_rc = rc + if main_rc is None: + main_rc = raw_codes[0] + + header = bytearray(4) header[0] = ord("M") header[1] = config.MPY_VERSION header[2] = config.native_arch << 2 | config.MICROPY_PY_BUILTINS_STR_UNICODE << 1 header[3] = config.mp_small_int_bits - header[4] = 32 # qstr_win_size merged_mpy.extend(header) + # Copy n_qstr, n_obj, qstr_table, obj_table from main_rc. + with open(main_rc.mpy_source_file, "rb") as f: + data = f.read(main_rc.raw_code_file_offset) + merged_mpy.extend(data[4:]) + bytecode = bytearray() - bytecode_len = 6 + len(raw_codes) * 5 + 2 - bytecode.append(bytecode_len << 2) # kind and length + bytecode_len = 3 + len(raw_codes) * 5 + 2 + bytecode.append(bytecode_len << 3 | 1 << 2) # kind, has_children and length bytecode.append(0b00000000) # signature prelude - bytecode.append(0b00001000) # size prelude - bytecode.extend(b"\x00\x01") # MP_QSTR_ - bytecode.extend(b"\x00\x01") # MP_QSTR_ + bytecode.append(0b00000010) # size prelude; n_info=1 + bytecode.extend(b"\x00") # simple_name: qstr index 0 (will use source filename) for idx in range(len(raw_codes)): bytecode.append(0x32) # MP_BC_MAKE_FUNCTION bytecode.append(idx) # index raw code bytecode.extend(b"\x34\x00\x59") # MP_BC_CALL_FUNCTION, 0 args, MP_BC_POP_TOP bytecode.extend(b"\x51\x63") # MP_BC_LOAD_NONE, MP_BC_RETURN_VALUE - bytecode.append(0) # n_obj - bytecode.append(len(raw_codes)) # n_raw_code - merged_mpy.extend(bytecode) + merged_mpy.append(len(raw_codes)) # n_children + for rc in raw_codes: with open(rc.mpy_source_file, "rb") as f: - f.read(4) # skip header - read_uint(f) # skip qstr_win_size + f.seek(rc.raw_code_file_offset) data = f.read() # read rest of mpy file merged_mpy.extend(data) @@ -978,7 +1513,12 @@ def main(): import argparse cmd_parser = argparse.ArgumentParser(description="A tool to work with MicroPython .mpy files.") - cmd_parser.add_argument("-d", "--dump", action="store_true", help="dump contents of files") + cmd_parser.add_argument( + "-x", "--hexdump", action="store_true", help="output an annotated hex dump of files" + ) + cmd_parser.add_argument( + "-d", "--disassemble", action="store_true", help="output disassembled contents of files" + ) cmd_parser.add_argument("-f", "--freeze", action="store_true", help="freeze files") cmd_parser.add_argument( "--merge", action="store_true", help="merge multiple .mpy files into one" @@ -1018,20 +1558,32 @@ def main(): else: config.MICROPY_QSTR_BYTES_IN_LEN = 1 config.MICROPY_QSTR_BYTES_IN_HASH = 1 - base_qstrs = {} + base_qstrs = list(qstrutil.static_qstr_list) + + # Load all .mpy files. + try: + compiled_modules = [read_mpy(file) for file in args.files] + except MPYReadError as er: + print(er, file=sys.stderr) + sys.exit(1) - raw_codes = [read_mpy(file) for file in args.files] + if args.hexdump: + hexdump_mpy(compiled_modules) - if args.dump: - dump_mpy(raw_codes) - elif args.freeze: + if args.disassemble: + if args.hexdump: + print() + disassemble_mpy(compiled_modules) + + if args.freeze: try: - freeze_mpy(base_qstrs, raw_codes) + freeze_mpy(base_qstrs, compiled_modules) except FreezeError as er: print(er, file=sys.stderr) sys.exit(1) - elif args.merge: - merged_mpy = merge_mpy(raw_codes, args.output) + + if args.merge: + merge_mpy(compiled_modules, args.output) if __name__ == "__main__": diff --git a/tools/mpy_ld.py b/tools/mpy_ld.py index 6bc1dbac05..0dac69e208 100755 --- a/tools/mpy_ld.py +++ b/tools/mpy_ld.py @@ -35,7 +35,7 @@ sys.path.append(os.path.dirname(__file__) + "/../py") import makeqstrdata as qstrutil # MicroPython constants -MPY_VERSION = 5 +MPY_VERSION = 6 MP_NATIVE_ARCH_X86 = 1 MP_NATIVE_ARCH_X64 = 2 MP_NATIVE_ARCH_ARMV7M = 5 @@ -860,11 +860,12 @@ class MPYOutput: def write_qstr(self, s): if s in qstrutil.static_qstr_list: - self.write_bytes(bytes([0, qstrutil.static_qstr_list.index(s) + 1])) + self.write_uint((qstrutil.static_qstr_list.index(s) + 1) << 1 | 1) else: s = bytes(s, "ascii") self.write_uint(len(s) << 1) self.write_bytes(s) + self.write_bytes(b"\x00") def write_reloc(self, base, offset, dest, n): need_offset = not (base == self.prev_base and offset == self.prev_offset + 1) @@ -905,14 +906,19 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): out.open(fmpy) # MPY: header - out.write_bytes( - bytearray( - [ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS, QSTR_WINDOW_SIZE] - ) - ) + out.write_bytes(bytearray([ord("M"), MPY_VERSION, env.arch.mpy_feature, MP_SMALL_INT_BITS])) + + # MPY: n_qstr + out.write_uint(1) + + # MPY: n_obj + out.write_uint(0) + + # MPY: qstr table + out.write_qstr(fmpy) # filename # MPY: kind/len - out.write_uint(len(env.full_text) << 2 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) + out.write_uint(len(env.full_text) << 3 | (MP_CODE_NATIVE_VIPER - MP_CODE_BYTECODE)) # MPY: machine code out.write_bytes(env.full_text) @@ -936,20 +942,15 @@ def build_mpy(env, entry_offset, fmpy, native_qstr_vals, native_qstr_objs): scope_flags |= MP_SCOPE_FLAG_VIPERBSS out.write_uint(scope_flags) - # MPY: n_obj - out.write_uint(0) - - # MPY: n_raw_code - out.write_uint(0) - - # MPY: rodata and/or bss + # MPY: bss and/or rodata if len(env.full_rodata): rodata_const_table_idx = 1 out.write_uint(len(env.full_rodata)) - out.write_bytes(env.full_rodata) if len(env.full_bss): - bss_const_table_idx = bool(env.full_rodata) + 1 + bss_const_table_idx = 2 out.write_uint(len(env.full_bss)) + if len(env.full_rodata): + out.write_bytes(env.full_rodata) # MPY: relocation information prev_kind = None |