UpsilonNumworks/Upsilon
1
0
Fork 0
mirror of https://github.com/UpsilonNumworks/Upsilon.git synced 2026-03-20 14:20:39 +01:00
Code Issues Packages Projects Releases Wiki Activity

[python] Update to MicroPython 1.9.4

Browse Source
This commit is contained in:
Romain Goyet
2018-05-23 11:35:29 +02:00
committed by EmilieNumworks
parent caff93cda0
commit 73250e727a
100 changed files with 2301 additions and 1417 deletions
Download Patch File Download Diff File Expand all files Collapse all files

326
python/src/py/parse.c
View File

@@ -3,7 +3,7 @@
*
* The MIT License (MIT)
*
* Copyright (c) 2013-2015 Damien P. George
* Copyright (c) 2013-2017 Damien P. George
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
@@ -58,15 +58,6 @@
// (un)comment to use rule names; for debugging
//#define USE_RULE_NAME (1)
typedef struct _rule_t {
byte rule_id;
byte act;
#ifdef USE_RULE_NAME
const char *rule_name;
#endif
uint16_t arg[];
} rule_t;
enum {
// define rules with a compile function
#define DEF_RULE(rule, comp, kind, ...) RULE_##rule,
@@ -84,6 +75,8 @@ enum {
#undef DEF_RULE_NC
};
// Define an array of actions corresponding to each rule
STATIC const uint8_t rule_act_table[] = {
#define or(n) (RULE_ACT_OR | n)
#define and(n) (RULE_ACT_AND | n)
#define and_ident(n) (RULE_ACT_AND | n | RULE_ACT_ALLOW_IDENT)
@@ -91,45 +84,129 @@ enum {
#define one_or_more (RULE_ACT_LIST | 2)
#define list (RULE_ACT_LIST | 1)
#define list_with_end (RULE_ACT_LIST | 3)
#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t)
#define rule(r) (RULE_ARG_RULE | RULE_##r)
#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r)
#ifdef USE_RULE_NAME
#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } };
#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } };
#else
#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } };
#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } };
#endif
#include "py/grammar.h"
#undef or
#undef and
#undef list
#undef list_with_end
#undef tok
#undef rule
#undef opt_rule
#undef one_or_more
#undef DEF_RULE
#undef DEF_RULE_NC
STATIC const rule_t *const rules[] = {
// define rules with a compile function
#define DEF_RULE(rule, comp, kind, ...) &rule_##rule,
#define DEF_RULE(rule, comp, kind, ...) kind,
#define DEF_RULE_NC(rule, kind, ...)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
NULL, // RULE_const_object
// define rules without a compile function
0, // RULE_const_object
#define DEF_RULE(rule, comp, kind, ...)
#define DEF_RULE_NC(rule, kind, ...) &rule_##rule,
#define DEF_RULE_NC(rule, kind, ...) kind,
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
#undef or
#undef and
#undef and_ident
#undef and_blank
#undef one_or_more
#undef list
#undef list_with_end
};
// Define the argument data for each rule, as a combined array
STATIC const uint16_t rule_arg_combined_table[] = {
#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t)
#define rule(r) (RULE_ARG_RULE | RULE_##r)
#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r)
#define DEF_RULE(rule, comp, kind, ...) __VA_ARGS__,
#define DEF_RULE_NC(rule, kind, ...)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
#define DEF_RULE(rule, comp, kind, ...)
#define DEF_RULE_NC(rule, kind, ...) __VA_ARGS__,
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
#undef tok
#undef rule
#undef opt_rule
};
// Macro to create a list of N identifiers where N is the number of variable arguments to the macro
#define RULE_EXPAND(x) x
#define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule))
#define RULE_PADDING2(rule, ...) RULE_EXPAND(RULE_PADDING3(rule, __VA_ARGS__))
#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, ...) __VA_ARGS__
#define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r,
// Use an enum to create constants specifying how much room a rule takes in rule_arg_combined_table
enum {
#define DEF_RULE(rule, comp, kind, ...) RULE_PADDING(rule, __VA_ARGS__)
#define DEF_RULE_NC(rule, kind, ...)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
#define DEF_RULE(rule, comp, kind, ...)
#define DEF_RULE_NC(rule, kind, ...) RULE_PADDING(rule, __VA_ARGS__)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
};
// Macro to compute the start of a rule in rule_arg_combined_table
#define RULE_ARG_OFFSET(rule, ...) RULE_ARG_OFFSET2(rule, __VA_ARGS__, RULE_ARG_OFFSET_IDS(rule))
#define RULE_ARG_OFFSET2(rule, ...) RULE_EXPAND(RULE_ARG_OFFSET3(rule, __VA_ARGS__))
#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) _13
#define RULE_ARG_OFFSET_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r,
// Use the above enum values to create a table of offsets for each rule's arg
// data, which indexes rule_arg_combined_table. The offsets require 9 bits of
// storage but only the lower 8 bits are stored here. The 9th bit is computed
// in get_rule_arg using the FIRST_RULE_WITH_OFFSET_ABOVE_255 constant.
STATIC const uint8_t rule_arg_offset_table[] = {
#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff,
#define DEF_RULE_NC(rule, kind, ...)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
0, // RULE_const_object
#define DEF_RULE(rule, comp, kind, ...)
#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff,
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
};
// Define a constant that's used to determine the 9th bit of the values in rule_arg_offset_table
static const size_t FIRST_RULE_WITH_OFFSET_ABOVE_255 =
#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) >= 0x100 ? RULE_##rule :
#define DEF_RULE_NC(rule, kind, ...)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
#define DEF_RULE(rule, comp, kind, ...)
#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) >= 0x100 ? RULE_##rule :
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
0;
#if USE_RULE_NAME
// Define an array of rule names corresponding to each rule
STATIC const char *const rule_name_table[] = {
#define DEF_RULE(rule, comp, kind, ...) #rule,
#define DEF_RULE_NC(rule, kind, ...)
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
"", // RULE_const_object
#define DEF_RULE(rule, comp, kind, ...)
#define DEF_RULE_NC(rule, kind, ...) #rule,
#include "py/grammar.h"
#undef DEF_RULE
#undef DEF_RULE_NC
};
#endif
typedef struct _rule_stack_t {
size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number
size_t rule_id : 8; // this must be large enough to fit largest rule number
@@ -164,6 +241,14 @@ typedef struct _parser_t {
#endif
} parser_t;
STATIC const uint16_t *get_rule_arg(uint8_t r_id) {
size_t off = rule_arg_offset_table[r_id];
if (r_id >= FIRST_RULE_WITH_OFFSET_ABOVE_255) {
off |= 0x100;
}
return &rule_arg_combined_table[off];
}
STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) {
// use a custom memory allocator to store parse nodes sequentially in large chunks
@@ -205,7 +290,7 @@ STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) {
return ret;
}
STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t arg_i) {
STATIC void push_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t arg_i) {
if (parser->rule_stack_top >= parser->rule_stack_alloc) {
rule_stack_t *rs = m_renew(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc, parser->rule_stack_alloc + MICROPY_ALLOC_PARSE_RULE_INC);
parser->rule_stack = rs;
@@ -213,21 +298,22 @@ STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, siz
}
rule_stack_t *rs = &parser->rule_stack[parser->rule_stack_top++];
rs->src_line = src_line;
rs->rule_id = rule->rule_id;
rs->rule_id = rule_id;
rs->arg_i = arg_i;
}
STATIC void push_rule_from_arg(parser_t *parser, size_t arg) {
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE);
size_t rule_id = arg & RULE_ARG_ARG_MASK;
push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0);
push_rule(parser, parser->lexer->tok_line, rule_id, 0);
}
STATIC void pop_rule(parser_t *parser, const rule_t **rule, size_t *arg_i, size_t *src_line) {
STATIC uint8_t pop_rule(parser_t *parser, size_t *arg_i, size_t *src_line) {
parser->rule_stack_top -= 1;
*rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id];
uint8_t rule_id = parser->rule_stack[parser->rule_stack_top].rule_id;
*arg_i = parser->rule_stack[parser->rule_stack_top].arg_i;
*src_line = parser->rule_stack[parser->rule_stack_top].src_line;
return rule_id;
}
bool mp_parse_node_is_const_false(mp_parse_node_t pn) {
@@ -313,11 +399,11 @@ void mp_parse_node_print(mp_parse_node_t pn, size_t indent) {
#endif
} else {
size_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns);
#ifdef USE_RULE_NAME
printf("%s(%u) (n=%u)\n", rules[MP_PARSE_NODE_STRUCT_KIND(pns)]->rule_name, (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n);
#else
#if USE_RULE_NAME
printf("%s(%u) (n=%u)\n", rule_name_table[MP_PARSE_NODE_STRUCT_KIND(pns)], (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n);
#else
printf("rule(%u) (n=%u)\n", (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n);
#endif
#endif
for (size_t i = 0; i < n; i++) {
mp_parse_node_print(pns->nodes[i], indent + 2);
}
@@ -369,7 +455,19 @@ STATIC mp_parse_node_t make_node_const_object(parser_t *parser, size_t src_line,
return (mp_parse_node_t)pn;
}
STATIC void push_result_token(parser_t *parser, const rule_t *rule) {
STATIC mp_parse_node_t mp_parse_node_new_small_int_checked(parser_t *parser, mp_obj_t o_val) {
(void)parser;
mp_int_t val = MP_OBJ_SMALL_INT_VALUE(o_val);
#if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D
// A parse node is only 32-bits and the small-int value must fit in 31-bits
if (((val ^ (val << 1)) & 0xffffffff80000000) != 0) {
return make_node_const_object(parser, 0, o_val);
}
#endif
return mp_parse_node_new_small_int(val);
}
STATIC void push_result_token(parser_t *parser, uint8_t rule_id) {
mp_parse_node_t pn;
mp_lexer_t *lex = parser->lexer;
if (lex->tok_kind == MP_TOKEN_NAME) {
@@ -377,10 +475,10 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) {
#if MICROPY_COMP_CONST
// if name is a standalone identifier, look it up in the table of dynamic constants
mp_map_elem_t *elem;
if (rule->rule_id == RULE_atom
if (rule_id == RULE_atom
&& (elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP)) != NULL) {
if (MP_OBJ_IS_SMALL_INT(elem->value)) {
pn = mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(elem->value));
pn = mp_parse_node_new_small_int_checked(parser, elem->value);
} else {
pn = make_node_const_object(parser, lex->tok_line, elem->value);
}
@@ -388,13 +486,13 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) {
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id);
}
#else
(void)rule;
(void)rule_id;
pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id);
#endif
} else if (lex->tok_kind == MP_TOKEN_INTEGER) {
mp_obj_t o = mp_parse_num_integer(lex->vstr.buf, lex->vstr.len, 0, lex);
if (MP_OBJ_IS_SMALL_INT(o)) {
pn = mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(o));
pn = mp_parse_node_new_small_int_checked(parser, o);
} else {
pn = make_node_const_object(parser, lex->tok_line, o);
}
@@ -417,7 +515,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) {
pn = mp_parse_node_new_leaf(lex->tok_kind == MP_TOKEN_STRING ? MP_PARSE_NODE_STRING : MP_PARSE_NODE_BYTES, qst);
} else {
// not interned, make a node holding a pointer to the string/bytes object
mp_obj_t o = mp_obj_new_str_of_type(
mp_obj_t o = mp_obj_new_str_copy(
lex->tok_kind == MP_TOKEN_STRING ? &mp_type_str : &mp_type_bytes,
(const byte*)lex->vstr.buf, lex->vstr.len);
pn = make_node_const_object(parser, lex->tok_line, o);
@@ -442,12 +540,12 @@ STATIC const mp_rom_map_elem_t mp_constants_table[] = {
STATIC MP_DEFINE_CONST_MAP(mp_constants_map, mp_constants_table);
#endif
STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t num_args);
STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args);
#if MICROPY_COMP_CONST_FOLDING
STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t *num_args) {
if (rule->rule_id == RULE_or_test
|| rule->rule_id == RULE_and_test) {
STATIC bool fold_logical_constants(parser_t *parser, uint8_t rule_id, size_t *num_args) {
if (rule_id == RULE_or_test
|| rule_id == RULE_and_test) {
// folding for binary logical ops: or and
size_t copy_to = *num_args;
for (size_t i = copy_to; i > 0;) {
@@ -457,7 +555,7 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t
// always need to keep the last value
break;
}
if (rule->rule_id == RULE_or_test) {
if (rule_id == RULE_or_test) {
if (mp_parse_node_is_const_true(pn)) {
//
break;
@@ -484,7 +582,7 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t
// we did a complete folding if there's only 1 arg left
return *num_args == 1;
} else if (rule->rule_id == RULE_not_test_2) {
} else if (rule_id == RULE_not_test_2) {
// folding for unary logical op: not
mp_parse_node_t pn = peek_result(parser, 0);
if (mp_parse_node_is_const_false(pn)) {
@@ -502,23 +600,23 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t
return false;
}
STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args) {
STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
// this code does folding of arbitrary integer expressions, eg 1 + 2 * 3 + 4
// it does not do partial folding, eg 1 + 2 + x -> 3 + x
mp_obj_t arg0;
if (rule->rule_id == RULE_expr
|| rule->rule_id == RULE_xor_expr
|| rule->rule_id == RULE_and_expr) {
if (rule_id == RULE_expr
|| rule_id == RULE_xor_expr
|| rule_id == RULE_and_expr) {
// folding for binary ops: | ^ &
mp_parse_node_t pn = peek_result(parser, num_args - 1);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
return false;
}
mp_binary_op_t op;
if (rule->rule_id == RULE_expr) {
if (rule_id == RULE_expr) {
op = MP_BINARY_OP_OR;
} else if (rule->rule_id == RULE_xor_expr) {
} else if (rule_id == RULE_xor_expr) {
op = MP_BINARY_OP_XOR;
} else {
op = MP_BINARY_OP_AND;
@@ -531,9 +629,9 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
}
arg0 = mp_binary_op(op, arg0, arg1);
}
} else if (rule->rule_id == RULE_shift_expr
|| rule->rule_id == RULE_arith_expr
|| rule->rule_id == RULE_term) {
} else if (rule_id == RULE_shift_expr
|| rule_id == RULE_arith_expr
|| rule_id == RULE_term) {
// folding for binary ops: << >> + - * / % //
mp_parse_node_t pn = peek_result(parser, num_args - 1);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
@@ -577,7 +675,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
}
arg0 = mp_binary_op(op, arg0, arg1);
}
} else if (rule->rule_id == RULE_factor_2) {
} else if (rule_id == RULE_factor_2) {
// folding for unary ops: + - ~
mp_parse_node_t pn = peek_result(parser, 0);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
@@ -596,7 +694,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
arg0 = mp_unary_op(op, arg0);
#if MICROPY_COMP_CONST
} else if (rule->rule_id == RULE_expr_stmt) {
} else if (rule_id == RULE_expr_stmt) {
mp_parse_node_t pn1 = peek_result(parser, 0);
if (!MP_PARSE_NODE_IS_NULL(pn1)
&& !(MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_augassign)
@@ -635,7 +733,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
if (qstr_str(id)[0] == '_') {
pop_result(parser); // pop const(value)
pop_result(parser); // pop id
push_result_rule(parser, 0, rules[RULE_pass_stmt], 0); // replace with "pass"
push_result_rule(parser, 0, RULE_pass_stmt, 0); // replace with "pass"
return true;
}
@@ -651,7 +749,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
#endif
#if MICROPY_COMP_MODULE_CONST
} else if (rule->rule_id == RULE_atom_expr_normal) {
} else if (rule_id == RULE_atom_expr_normal) {
mp_parse_node_t pn0 = peek_result(parser, 1);
mp_parse_node_t pn1 = peek_result(parser, 0);
if (!(MP_PARSE_NODE_IS_ID(pn0)
@@ -686,7 +784,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
pop_result(parser);
}
if (MP_OBJ_IS_SMALL_INT(arg0)) {
push_result_node(parser, mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(arg0)));
push_result_node(parser, mp_parse_node_new_small_int_checked(parser, arg0));
} else {
// TODO reuse memory for parse node struct?
push_result_node(parser, make_node_const_object(parser, 0, arg0));
@@ -696,9 +794,9 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args
}
#endif
STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t num_args) {
STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args) {
// optimise away parenthesis around an expression if possible
if (rule->rule_id == RULE_atom_paren) {
if (rule_id == RULE_atom_paren) {
// there should be just 1 arg for this rule
mp_parse_node_t pn = peek_result(parser, 0);
if (MP_PARSE_NODE_IS_NULL(pn)) {
@@ -712,11 +810,11 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *ru
}
#if MICROPY_COMP_CONST_FOLDING
if (fold_logical_constants(parser, rule, &num_args)) {
if (fold_logical_constants(parser, rule_id, &num_args)) {
// we folded this rule so return straight away
return;
}
if (fold_constants(parser, rule, num_args)) {
if (fold_constants(parser, rule_id, num_args)) {
// we folded this rule so return straight away
return;
}
@@ -724,7 +822,7 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *ru
mp_parse_node_struct_t *pn = parser_alloc(parser, sizeof(mp_parse_node_struct_t) + sizeof(mp_parse_node_t) * num_args);
pn->source_line = src_line;
pn->kind_num_nodes = (rule->rule_id & 0xff) | (num_args << 8);
pn->kind_num_nodes = (rule_id & 0xff) | (num_args << 8);
for (size_t i = num_args; i > 0; i--) {
pn->nodes[i - 1] = pop_result(parser);
}
@@ -761,14 +859,11 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break;
default: top_level_rule = RULE_file_input;
}
push_rule(&parser, lex->tok_line, rules[top_level_rule], 0);
push_rule(&parser, lex->tok_line, top_level_rule, 0);
// parse!
size_t n, i; // state for the current rule
size_t rule_src_line; // source line for the first token matched by the current rule
bool backtrack = false;
const rule_t *rule = NULL;
for (;;) {
next_rule:
@@ -776,19 +871,24 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
break;
}
pop_rule(&parser, &rule, &i, &rule_src_line);
n = rule->act & RULE_ACT_ARG_MASK;
// Pop the next rule to process it
size_t i; // state for the current rule
size_t rule_src_line; // source line for the first token matched by the current rule
uint8_t rule_id = pop_rule(&parser, &i, &rule_src_line);
uint8_t rule_act = rule_act_table[rule_id];
const uint16_t *rule_arg = get_rule_arg(rule_id);
size_t n = rule_act & RULE_ACT_ARG_MASK;
/*
#if 0
// debugging
printf("depth=%d ", parser.rule_stack_top);
printf("depth=" UINT_FMT " ", parser.rule_stack_top);
for (int j = 0; j < parser.rule_stack_top; ++j) {
printf(" ");
}
printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack);
*/
printf("%s n=" UINT_FMT " i=" UINT_FMT " bt=%d\n", rule_name_table[rule_id], n, i, backtrack);
#endif
switch (rule->act & RULE_ACT_KIND_MASK) {
switch (rule_act & RULE_ACT_KIND_MASK) {
case RULE_ACT_OR:
if (i > 0 && !backtrack) {
goto next_rule;
@@ -796,19 +896,19 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
backtrack = false;
}
for (; i < n; ++i) {
uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK;
uint16_t kind = rule_arg[i] & RULE_ARG_KIND_MASK;
if (kind == RULE_ARG_TOK) {
if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, rule);
if (lex->tok_kind == (rule_arg[i] & RULE_ARG_ARG_MASK)) {
push_result_token(&parser, rule_id);
mp_lexer_to_next(lex);
goto next_rule;
}
} else {
assert(kind == RULE_ARG_RULE);
if (i + 1 < n) {
push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule
push_rule(&parser, rule_src_line, rule_id, i + 1); // save this or-rule
}
push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule
push_rule_from_arg(&parser, rule_arg[i]); // push child of or-rule
goto next_rule;
}
}
@@ -820,7 +920,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// failed, backtrack if we can, else syntax error
if (backtrack) {
assert(i > 0);
if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
if ((rule_arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) {
// an optional rule that failed, so continue with next arg
push_result_node(&parser, MP_PARSE_NODE_NULL);
backtrack = false;
@@ -837,13 +937,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// progress through the rule
for (; i < n; ++i) {
if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if ((rule_arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// need to match a token
mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK;
mp_token_kind_t tok_kind = rule_arg[i] & RULE_ARG_ARG_MASK;
if (lex->tok_kind == tok_kind) {
// matched token
if (tok_kind == MP_TOKEN_NAME) {
push_result_token(&parser, rule);
push_result_token(&parser, rule_id);
}
mp_lexer_to_next(lex);
} else {
@@ -858,8 +958,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
}
} else {
push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule
push_rule(&parser, rule_src_line, rule_id, i + 1); // save this and-rule
push_rule_from_arg(&parser, rule_arg[i]); // push child of and-rule
goto next_rule;
}
}
@@ -870,7 +970,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
#if !MICROPY_ENABLE_DOC_STRING
// this code discards lonely statements, such as doc strings
if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
if (input_kind != MP_PARSE_SINGLE_INPUT && rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) {
mp_parse_node_t p = peek_result(&parser, 1);
if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p))
|| MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_const_object)) {
@@ -879,7 +979,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// Pushing the "pass" rule here will overwrite any RULE_const_object
// entry that was on the result stack, allowing the GC to reclaim
// the memory from the const object when needed.
push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0);
push_result_rule(&parser, rule_src_line, RULE_pass_stmt, 0);
break;
}
}
@@ -890,8 +990,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
size_t num_not_nil = 0;
for (size_t x = n; x > 0;) {
--x;
if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK;
if ((rule_arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
mp_token_kind_t tok_kind = rule_arg[x] & RULE_ARG_ARG_MASK;
if (tok_kind == MP_TOKEN_NAME) {
// only tokens which were names are pushed to stack
i += 1;
@@ -906,7 +1006,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
}
if (num_not_nil == 1 && (rule->act & RULE_ACT_ALLOW_IDENT)) {
if (num_not_nil == 1 && (rule_act & RULE_ACT_ALLOW_IDENT)) {
// this rule has only 1 argument and should not be emitted
mp_parse_node_t pn = MP_PARSE_NODE_NULL;
for (size_t x = 0; x < i; ++x) {
@@ -919,19 +1019,19 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
} else {
// this rule must be emitted
if (rule->act & RULE_ACT_ADD_BLANK) {
if (rule_act & RULE_ACT_ADD_BLANK) {
// and add an extra blank node at the end (used by the compiler to store data)
push_result_node(&parser, MP_PARSE_NODE_NULL);
i += 1;
}
push_result_rule(&parser, rule_src_line, rule, i);
push_result_rule(&parser, rule_src_line, rule_id, i);
}
break;
}
default: {
assert((rule->act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST);
assert((rule_act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST);
// n=2 is: item item*
// n=1 is: item (sep item)*
@@ -969,13 +1069,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
} else {
for (;;) {
size_t arg = rule->arg[i & 1 & n];
size_t arg = rule_arg[i & 1 & n];
if ((arg & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) {
if (i & 1 & n) {
// separators which are tokens are not pushed to result stack
} else {
push_result_token(&parser, rule);
push_result_token(&parser, rule_id);
}
mp_lexer_to_next(lex);
// got element of list, so continue parsing list
@@ -988,7 +1088,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
}
} else {
assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE);
push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule
push_rule(&parser, rule_src_line, rule_id, i + 1); // save this list-rule
push_rule_from_arg(&parser, arg); // push child of list-rule
goto next_rule;
}
@@ -998,7 +1098,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// compute number of elements in list, result in i
i -= 1;
if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
if ((n & 1) && (rule_arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) {
// don't count separators when they are tokens
i = (i + 1) / 2;
}
@@ -1007,12 +1107,12 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) {
// list matched single item
if (had_trailing_sep) {
// if there was a trailing separator, make a list of a single item
push_result_rule(&parser, rule_src_line, rule, i);
push_result_rule(&parser, rule_src_line, rule_id, i);
} else {
// just leave single item on stack (ie don't wrap in a list)
}
} else {
push_result_rule(&parser, rule_src_line, rule, i);
push_result_rule(&parser, rule_src_line, rule_id, i);
}
break;
}