py: Add ability to have frozen persistent bytecode from .mpy files.

The config variable MICROPY_MODULE_FROZEN is now made of two separate
parts: MICROPY_MODULE_FROZEN_STR and MICROPY_MODULE_FROZEN_MPY.  This
allows to have none, either or both of frozen strings and frozen mpy
files (aka frozen bytecode).

7 files changed, 105 insertions, 27 deletions

diff --git a/py/builtinimport.c b/py/builtinimport.c
index ec79357cb7..0e4dce6430 100644
--- a/py/builtinimport.c
+++ b/py/builtinimport.c
@@ -144,7 +144,7 @@ STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex, const char
 }
 #endif
 
-#if MICROPY_PERSISTENT_CODE_LOAD
+#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_MODULE_FROZEN_MPY
 STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
     #if MICROPY_PY___FILE__
     // TODO
@@ -182,8 +182,9 @@ STATIC void do_execute_raw_code(mp_obj_t module_obj, mp_raw_code_t *raw_code) {
 #endif
 
 STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
-    // create the lexer
+    #if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_ENABLE_COMPILER
     char *file_str = vstr_null_terminated_str(file);
+    #endif
 
     #if MICROPY_PERSISTENT_CODE_LOAD
     if (file_str[file->len - 3] == 'm') {
@@ -340,8 +341,9 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
     DEBUG_printf("Module not yet loaded\n");
 
     #if MICROPY_MODULE_FROZEN
-    mp_lexer_t *lex = mp_find_frozen_module(mod_str, mod_len);
-    if (lex != NULL) {
+    void *frozen_data;
+    int frozen_type = mp_find_frozen_module(mod_str, mod_len, &frozen_data);
+    if (frozen_type != MP_FROZEN_NONE) {
         module_obj = mp_obj_new_module(module_name_qstr);
         // if args[3] (fromtuple) has magic value False, set up
         // this module for command-line "-m" option (set module's
@@ -351,7 +353,16 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
             mp_obj_module_t *o = MP_OBJ_TO_PTR(module_obj);
             mp_obj_dict_store(MP_OBJ_FROM_PTR(o->globals), MP_OBJ_NEW_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR___main__));
         }
-        do_load_from_lexer(module_obj, lex, mod_str);
+        #if MICROPY_MODULE_FROZEN_STR
+        if (frozen_type == MP_FROZEN_STR) {
+            do_load_from_lexer(module_obj, frozen_data, mod_str);
+        }
+        #endif
+        #if MICROPY_MODULE_FROZEN_MPY
+        if (frozen_type == MP_FROZEN_MPY) {
+            do_execute_raw_code(module_obj, frozen_data);
+        }
+        #endif
         return module_obj;
     }
     #endif
diff --git a/py/emitglue.c b/py/emitglue.c
index 7e5edce756..c657d40d12 100644
--- a/py/emitglue.c
+++ b/py/emitglue.c
@@ -172,7 +172,7 @@ mp_obj_t mp_make_closure_from_raw_code(const mp_raw_code_t *rc, mp_uint_t n_clos
     return mp_obj_new_closure(ffun, n_closed_over & 0xff, args + ((n_closed_over >> 7) & 2));
 }
 
-#if MICROPY_PERSISTENT_CODE
+#if MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
 #include "py/smallint.h"
 
@@ -229,7 +229,7 @@ STATIC void extract_prelude(const byte **ip, const byte **ip2, bytecode_prelude_
     }
 }
 
-#endif // MICROPY_PERSISTENT_CODE
+#endif // MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE
 
 #if MICROPY_PERSISTENT_CODE_LOAD
 
diff --git a/py/frozenmod.c b/py/frozenmod.c
index 6b76bf662b..92afdb9a5e 100644
--- a/py/frozenmod.c
+++ b/py/frozenmod.c
@@ -4,6 +4,7 @@
  * The MIT License (MIT)
  *
  * Copyright (c) 2015 Paul Sokolovsky
+ * Copyright (c) 2016 Damien P. George
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -30,7 +31,7 @@
 #include "py/lexer.h"
 #include "py/frozenmod.h"
 
-#if MICROPY_MODULE_FROZEN
+#if MICROPY_MODULE_FROZEN_STR
 
 #ifndef MICROPY_MODULE_FROZEN_LEXER
 #define MICROPY_MODULE_FROZEN_LEXER mp_lexer_new_from_str_len
@@ -38,24 +39,67 @@
 mp_lexer_t *MICROPY_MODULE_FROZEN_LEXER(qstr src_name, const char *str, mp_uint_t len, mp_uint_t free_len);
 #endif
 
-extern const char mp_frozen_names[];
-extern const uint32_t mp_frozen_sizes[];
-extern const char mp_frozen_content[];
+extern const char mp_frozen_str_names[];
+extern const uint32_t mp_frozen_str_sizes[];
+extern const char mp_frozen_str_content[];
 
-mp_lexer_t *mp_find_frozen_module(const char *str, int len) {
-    const char *name = mp_frozen_names;
+STATIC mp_lexer_t *mp_find_frozen_str(const char *str, size_t len) {
+    const char *name = mp_frozen_str_names;
 
     size_t offset = 0;
     for (int i = 0; *name != 0; i++) {
-        int l = strlen(name);
+        size_t l = strlen(name);
         if (l == len && !memcmp(str, name, l)) {
-            mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_content + offset, mp_frozen_sizes[i], 0);
+            mp_lexer_t *lex = MICROPY_MODULE_FROZEN_LEXER(MP_QSTR_, mp_frozen_str_content + offset, mp_frozen_str_sizes[i], 0);
             return lex;
         }
         name += l + 1;
-        offset += mp_frozen_sizes[i] + 1;
+        offset += mp_frozen_str_sizes[i] + 1;
+    }
+    return NULL;
+}
+
+#endif
+
+#if MICROPY_MODULE_FROZEN_MPY
+
+#include "py/emitglue.h"
+
+extern const char mp_frozen_mpy_names[];
+extern const mp_raw_code_t *const mp_frozen_mpy_content[];
+
+STATIC const mp_raw_code_t *mp_find_frozen_mpy(const char *str, size_t len) {
+    const char *name = mp_frozen_mpy_names;
+    for (size_t i = 0; *name != 0; i++) {
+        size_t l = strlen(name);
+        if (l == len && !memcmp(str, name, l)) {
+            return mp_frozen_mpy_content[i];
+        }
+        name += l + 1;
     }
     return NULL;
 }
 
-#endif // MICROPY_MODULE_FROZEN
+#endif
+
+#if MICROPY_MODULE_FROZEN
+
+int mp_find_frozen_module(const char *str, size_t len, void **data) {
+    #if MICROPY_MODULE_FROZEN_STR
+    mp_lexer_t *lex = mp_find_frozen_str(str, len);
+    if (lex != NULL) {
+        *data = lex;
+        return MP_FROZEN_STR;
+    }
+    #endif
+    #if MICROPY_MODULE_FROZEN_MPY
+    const mp_raw_code_t *rc = mp_find_frozen_mpy(str, len);
+    if (rc != NULL) {
+        *data = (void*)rc;
+        return MP_FROZEN_MPY;
+    }
+    #endif
+    return MP_FROZEN_NONE;
+}
+
+#endif
diff --git a/py/frozenmod.h b/py/frozenmod.h
index 67caced14d..a1638d2293 100644
--- a/py/frozenmod.h
+++ b/py/frozenmod.h
@@ -24,4 +24,10 @@
  * THE SOFTWARE.
  */
 
-mp_lexer_t *mp_find_frozen_module(const char *str, int len);
+enum {
+    MP_FROZEN_NONE,
+    MP_FROZEN_STR,
+    MP_FROZEN_MPY,
+};
+
+int mp_find_frozen_module(const char *str, size_t len, void **data);
diff --git a/py/mpconfig.h b/py/mpconfig.h
index 77ac614939..42ef19b72c 100644
--- a/py/mpconfig.h
+++ b/py/mpconfig.h
@@ -234,7 +234,7 @@
 // Whether generated code can persist independently of the VM/runtime instance
 // This is enabled automatically when needed by other features
 #ifndef MICROPY_PERSISTENT_CODE
-#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE)
+#define MICROPY_PERSISTENT_CODE (MICROPY_PERSISTENT_CODE_LOAD || MICROPY_PERSISTENT_CODE_SAVE || MICROPY_MODULE_FROZEN_MPY)
 #endif
 
 // Whether to emit x64 native code
@@ -526,9 +526,19 @@ typedef double mp_float_t;
 #define MICROPY_MODULE_WEAK_LINKS (0)
 #endif
 
-// Whether frozen modules are supported
+// Whether frozen modules are supported in the form of strings
+#ifndef MICROPY_MODULE_FROZEN_STR
+#define MICROPY_MODULE_FROZEN_STR (0)
+#endif
+
+// Whether frozen modules are supported in the form of .mpy files
+#ifndef MICROPY_MODULE_FROZEN_MPY
+#define MICROPY_MODULE_FROZEN_MPY (0)
+#endif
+
+// Convenience macro for whether frozen modules are supported
 #ifndef MICROPY_MODULE_FROZEN
-#define MICROPY_MODULE_FROZEN (0)
+#define MICROPY_MODULE_FROZEN (MICROPY_MODULE_FROZEN_STR || MICROPY_MODULE_FROZEN_MPY)
 #endif
 
 // Whether you can override builtins in the builtins module
diff --git a/py/qstr.c b/py/qstr.c
index 4268946fbe..ef31a682ee 100644
--- a/py/qstr.c
+++ b/py/qstr.c
@@ -87,11 +87,11 @@ mp_uint_t qstr_compute_hash(const byte *data, size_t len) {
     return hash;
 }
 
-STATIC const qstr_pool_t const_pool = {
+const qstr_pool_t mp_qstr_const_pool = {
     NULL,               // no previous pool
     0,                  // no previous pool
     10,                 // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
-    MP_QSTR_number_of,  // corresponds to number of strings in array just below
+    MP_QSTRnumber_of,   // corresponds to number of strings in array just below
     {
 #define QDEF(id, str) str,
 #include "genhdr/qstrdefs.generated.h"
@@ -99,8 +99,15 @@ STATIC const qstr_pool_t const_pool = {
     },
 };
 
+#ifdef MICROPY_QSTR_EXTRA_POOL
+extern const qstr_pool_t MICROPY_QSTR_EXTRA_POOL;
+#define CONST_POOL MICROPY_QSTR_EXTRA_POOL
+#else
+#define CONST_POOL mp_qstr_const_pool
+#endif
+
 void qstr_init(void) {
-    MP_STATE_VM(last_pool) = (qstr_pool_t*)&const_pool; // we won't modify the const_pool since it has no allocated room left
+    MP_STATE_VM(last_pool) = (qstr_pool_t*)&CONST_POOL; // we won't modify the const_pool since it has no allocated room left
     MP_STATE_VM(qstr_last_chunk) = NULL;
 }
 
@@ -258,7 +265,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
     *n_qstr = 0;
     *n_str_data_bytes = 0;
     *n_total_bytes = 0;
-    for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) {
+    for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
         *n_pool += 1;
         *n_qstr += pool->len;
         for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
@@ -275,7 +282,7 @@ void qstr_pool_info(size_t *n_pool, size_t *n_qstr, size_t *n_str_data_bytes, si
 
 #if MICROPY_PY_MICROPYTHON_MEM_INFO
 void qstr_dump_data(void) {
-    for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &const_pool; pool = pool->prev) {
+    for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL && pool != &CONST_POOL; pool = pool->prev) {
         for (const byte **q = pool->qstrs, **q_top = pool->qstrs + pool->len; q < q_top; q++) {
             mp_printf(&mp_plat_print, "Q(%s)\n", Q_GET_DATA(*q));
         }
diff --git a/py/qstr.h b/py/qstr.h
index b68f60a443..b5c261f0d5 100644
--- a/py/qstr.h
+++ b/py/qstr.h
@@ -40,7 +40,7 @@ enum {
 #define QDEF(id, str) id,
 #include "genhdr/qstrdefs.generated.h"
 #undef QDEF
-    MP_QSTR_number_of,
+    MP_QSTRnumber_of, // no underscore so it can't clash with any of the above
 };
 
 typedef size_t qstr;