Abstract out back-end stream functionality from lexer.

5 files changed, 68 insertions, 5 deletions

diff --git a/unix/Makefile b/unix/Makefile
index a2c9b9f5fd..7c8b5a2b9a 100644
--- a/unix/Makefile
+++ b/unix/Makefile
@@ -7,14 +7,15 @@ LDFLAGS =
 
 SRC_C = \
 	main.c \
+	lexerunix.c \
 
 PY_O = \
 	nlrx64.o \
 	malloc.o \
 	qstr.o \
+	vstr.o \
 	misc.o \
 	lexer.o \
-	lexerfile.o \
 	parse.o \
 	scope.o \
 	compile.o \
diff --git a/unix/lexerunix.c b/unix/lexerunix.c
new file mode 100644
index 0000000000..617d92bb82
--- /dev/null
+++ b/unix/lexerunix.c
@@ -0,0 +1,55 @@
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include "misc.h"
+#include "lexer.h"
+
+typedef struct _str_buf_t {
+    bool free;                  // free src_beg when done
+    const char *src_beg;        // beginning of source
+    const char *src_cur;        // current location in source
+    const char *src_end;        // end (exclusive) of source
+} str_buf_t;
+
+unichar str_buf_next_char(str_buf_t *sb) {
+    if (sb->src_cur < sb->src_end) {
+        return *sb->src_cur++;
+    } else {
+        return PY_LEXER_CHAR_EOF;
+    }
+}
+
+void str_buf_free(str_buf_t *sb) {
+    if (sb) {
+        if (sb->free) {
+            m_free((char*)sb->src_beg);
+        }
+        m_free(sb);
+    }
+}
+
+py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str) {
+    str_buf_t *sb = m_new(str_buf_t, 1);
+    sb->free = free_str;
+    sb->src_beg = str;
+    sb->src_cur = str;
+    sb->src_end = str + len;
+    return py_lexer_new(src_name, sb, (py_lexer_stream_next_char_t)str_buf_next_char, (py_lexer_stream_free_t)str_buf_free);
+}
+
+py_lexer_t *py_lexer_new_from_file(const char *filename) {
+    int fd = open(filename, O_RDONLY);
+    if (fd < 0) {
+        printf("cannot open file %s\n", filename);
+        return NULL;
+    }
+    uint size = lseek(fd, 0, SEEK_END);
+    lseek(fd, 0, SEEK_SET);
+    char *data = m_new(char, size);
+    read(fd, data, size);
+    close(fd);
+
+    return py_lexer_new_from_str_len(filename, data, size, true);
+}
diff --git a/unix/lexerunix.h b/unix/lexerunix.h
new file mode 100644
index 0000000000..aa7631cb0b
--- /dev/null
+++ b/unix/lexerunix.h
@@ -0,0 +1,2 @@
+py_lexer_t *py_lexer_new_from_str_len(const char *src_name, const char *str, uint len, bool free_str);
+py_lexer_t *py_lexer_new_from_file(const char *filename);
diff --git a/unix/main.c b/unix/main.c
index 018e1a9704..e3999db68b 100644
--- a/unix/main.c
+++ b/unix/main.c
@@ -6,6 +6,7 @@
 #include "misc.h"
 #include "mpyconfig.h"
 #include "lexer.h"
+#include "lexerunix.h"
 #include "parse.h"
 #include "compile.h"
 #include "runtime.h"
@@ -67,7 +68,7 @@ void do_repl() {
                 line = line3;
             }
         }
-        py_lexer_t *lex = py_lexer_from_str_len("<stdin>", line, strlen(line), false);
+        py_lexer_t *lex = py_lexer_new_from_str_len("<stdin>", line, strlen(line), false);
         py_parse_node_t pn = py_parse(lex, PY_PARSE_SINGLE_INPUT);
         if (pn != PY_PARSE_NODE_NULL) {
             //py_parse_node_show(pn, 0);
@@ -91,7 +92,7 @@ void do_repl() {
 }
 
 void do_file(const char *file) {
-    py_lexer_t *lex = py_lexer_from_file(file);
+    py_lexer_t *lex = py_lexer_new_from_file(file);
     //const char *pysrc = "def f():\n  x=x+1\n  print(42)\n";
     //py_lexer_t *lex = py_lexer_from_str_len("<>", pysrc, strlen(pysrc), false);
     if (lex == NULL) {
@@ -119,7 +120,11 @@ void do_file(const char *file) {
 
             py_lexer_free(lex);
 
-#if !MICROPY_EMIT_CPYTHON
+#if MICROPY_EMIT_CPYTHON
+            if (!comp_ok) {
+                printf("compile error\n");
+            }
+#else
             if (1 && comp_ok) {
                 // execute it
                 py_obj_t module_fun = rt_make_function_from_id(1);
diff --git a/unix/mpyconfig.h b/unix/mpyconfig.h
index 3ab17e6cab..587b09b16e 100644
--- a/unix/mpyconfig.h
+++ b/unix/mpyconfig.h
@@ -1,7 +1,7 @@
 // options to control how Micro Python is built
 
 #define MICROPY_ENABLE_FLOAT        (1)
-#define MICROPY_EMIT_CPYTHON        (0)
+#define MICROPY_EMIT_CPYTHON        (1)
 #define MICROPY_EMIT_X64            (1)
 #define MICROPY_EMIT_THUMB          (0)
 #define MICROPY_EMIT_INLINE_THUMB   (0)