py: Improve memory management for parser; add lexer error for bad line cont.

1 files changed, 34 insertions, 7 deletions

diff --git a/py/parse.c b/py/parse.c
index d3786ba956..a619c90507 100644
--- a/py/parse.c
+++ b/py/parse.c
@@ -88,6 +88,7 @@ typedef struct _parser_t {
     uint rule_stack_top;
     rule_stack_t *rule_stack;
 
+    uint result_stack_alloc;
     uint result_stack_top;
     mp_parse_node_t *result_stack;
 } parser_t;
@@ -121,7 +122,7 @@ mp_parse_node_t mp_parse_node_new_leaf(machine_int_t kind, machine_int_t arg) {
 
 int num_parse_nodes_allocated = 0;
 mp_parse_node_struct_t *parse_node_new_struct(int rule_id, int num_args) {
-    mp_parse_node_struct_t *pn = m_malloc(sizeof(mp_parse_node_struct_t) + num_args * sizeof(mp_parse_node_t));
+    mp_parse_node_struct_t *pn = m_new_obj_var(mp_parse_node_struct_t, mp_parse_node_t, num_args);
     pn->source = 0; // TODO
     pn->kind_num_nodes = (rule_id & 0xff) | (num_args << 8);
     num_parse_nodes_allocated += 1;
@@ -180,6 +181,10 @@ static mp_parse_node_t peek_result(parser_t *parser, int pos) {
 }
 
 static void push_result_node(parser_t *parser, mp_parse_node_t pn) {
+    if (parser->result_stack_top >= parser->result_stack_alloc) {
+        parser->result_stack = m_renew(mp_parse_node_t, parser->result_stack, parser->result_stack_alloc, parser->result_stack_alloc * 2);
+        parser->result_stack_alloc *= 2;
+    }
     parser->result_stack[parser->result_stack_top++] = pn;
 }
 
@@ -252,14 +257,20 @@ static void push_result_rule(parser_t *parser, const rule_t *rule, int num_args)
 }
 
 mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
-    parser_t *parser = m_new(parser_t, 1);
+
+    // allocate memory for the parser and its stacks
+
+    parser_t *parser = m_new_obj(parser_t);
+
     parser->rule_stack_alloc = 64;
     parser->rule_stack_top = 0;
     parser->rule_stack = m_new(rule_stack_t, parser->rule_stack_alloc);
 
-    parser->result_stack = m_new(mp_parse_node_t, 1000);
+    parser->result_stack_alloc = 64;
     parser->result_stack_top = 0;
+    parser->result_stack = m_new(mp_parse_node_t, parser->result_stack_alloc);
 
+    // work out the top-level rule to use, and push it on the stack
     int top_level_rule;
     switch (input_kind) {
         case MP_PARSE_SINGLE_INPUT: top_level_rule = RULE_single_input; break;
@@ -268,6 +279,8 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
     }
     push_rule(parser, rules[top_level_rule], 0);
 
+    // parse!
+
     uint n, i;
     bool backtrack = false;
     const rule_t *rule;
@@ -558,12 +571,25 @@ mp_parse_node_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
 
     //printf("--------------\n");
     //result_stack_show(parser);
-    assert(parser->result_stack_top == 1);
-    //printf("maximum depth: %d\n", parser->rule_stack_alloc);
+    //printf("rule stack alloc: %d\n", parser->rule_stack_alloc);
+    //printf("result stack alloc: %d\n", parser->result_stack_alloc);
     //printf("number of parse nodes allocated: %d\n", num_parse_nodes_allocated);
-    return parser->result_stack[0];
+
+    // get the root parse node that we created
+    assert(parser->result_stack_top == 1);
+    mp_parse_node_t result = parser->result_stack[0];
+
+finished:
+    // free the memory that we don't need anymore
+    m_del(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc);
+    m_del(mp_parse_node_t, parser->result_stack, parser->result_stack_alloc);
+    m_del_obj(parser_t, parser);
+
+    // return the result
+    return result;
 
 syntax_error:
+    // TODO these should raise a proper exception
     if (mp_lexer_is_kind(lex, MP_TOKEN_INDENT)) {
         mp_lexer_show_error_pythonic(lex, "IndentationError: unexpected indent");
     } else if (mp_lexer_is_kind(lex, MP_TOKEN_DEDENT_MISMATCH)) {
@@ -575,5 +601,6 @@ syntax_error:
 #endif
         mp_token_show(mp_lexer_cur(lex));
     }
-    return MP_PARSE_NODE_NULL;
+    result = MP_PARSE_NODE_NULL;
+    goto finished;
 }