From 73250e727a0e102db2b6607a3b68852f08d360da Mon Sep 17 00:00:00 2001 From: Romain Goyet Date: Wed, 23 May 2018 11:35:29 +0200 Subject: [PATCH] [python] Update to MicroPython 1.9.4 --- python/Makefile | 108 ++++------ python/port/builtins.c | 2 +- python/port/genhdr/mpversion.h | 4 +- python/port/genhdr/qstrdefs.in.h | 54 +++-- python/src/extmod/virtpin.h | 4 + python/src/py/asmarm.c | 5 +- python/src/py/asmarm.h | 15 +- python/src/py/asmbase.c | 6 +- python/src/py/asmthumb.c | 5 +- python/src/py/asmthumb.h | 16 +- python/src/py/asmx64.c | 4 +- python/src/py/asmx64.h | 15 +- python/src/py/asmx86.c | 3 +- python/src/py/asmx86.h | 17 +- python/src/py/asmxtensa.h | 15 +- python/src/py/binary.c | 2 +- python/src/py/builtinhelp.c | 4 +- python/src/py/builtinimport.c | 47 ++++- python/src/py/compile.c | 101 +++++----- python/src/py/emitbc.c | 6 +- python/src/py/emitglue.c | 5 +- python/src/py/emitglue.h | 7 +- python/src/py/emitnarm.c | 15 ++ python/src/py/emitnative.c | 200 ++++++------------- python/src/py/emitnthumb.c | 15 ++ python/src/py/emitnx64.c | 15 ++ python/src/py/emitnx86.c | 68 +++++++ python/src/py/emitnxtensa.c | 15 ++ python/src/py/formatfloat.c | 10 +- python/src/py/gc.c | 111 ++++++----- python/src/py/makeqstrdata.py | 3 + python/src/py/malloc.c | 18 +- python/src/py/map.c | 14 ++ python/src/py/misc.h | 24 +-- python/src/py/modbuiltins.c | 141 ++++++------- python/src/py/modcollections.c | 3 + python/src/py/modio.c | 6 +- python/src/py/modmicropython.c | 22 ++- python/src/py/modsys.c | 4 +- python/src/py/modthread.c | 6 + python/src/py/mpconfig.h | 87 ++++++++- python/src/py/mperrno.h | 2 +- python/src/py/mpprint.c | 28 ++- python/src/py/mpstate.h | 14 +- python/src/py/mpz.c | 145 ++++++-------- python/src/py/mpz.h | 4 + python/src/py/nlr.c | 51 +++++ python/src/py/nlr.h | 86 +++++--- python/src/py/nlrsetjmp.c | 7 +- python/src/py/nlrthumb.c | 39 ++-- python/src/py/nlrx64.c | 35 +--- python/src/py/nlrx86.c | 33 +--- python/src/py/nlrxtensa.c | 25 +-- python/src/py/obj.h | 28 +-- python/src/py/objarray.c | 3 +- python/src/py/objboundmeth.c | 8 + python/src/py/objdeque.c | 167 ++++++++++++++++ python/src/py/objdict.c | 58 +++--- python/src/py/objexcept.c | 36 +--- python/src/py/objexcept.h | 13 ++ python/src/py/objfloat.c | 14 +- python/src/py/objfun.c | 64 +++--- python/src/py/objgenerator.c | 35 +++- python/src/py/objint.c | 23 ++- python/src/py/objint_longlong.c | 14 +- python/src/py/objint_mpz.c | 8 +- python/src/py/objlist.h | 2 + python/src/py/objmodule.c | 28 +-- python/src/py/objmodule.h | 9 + python/src/py/objnamedtuple.c | 56 ++++-- python/src/py/objnamedtuple.h | 44 +++++ python/src/py/objobject.c | 9 +- python/src/py/objrange.c | 21 ++ python/src/py/objset.c | 8 +- python/src/py/objstr.c | 94 ++++++--- python/src/py/objstr.h | 1 + python/src/py/objstringio.c | 30 ++- python/src/py/objstrunicode.c | 6 +- python/src/py/objtype.c | 173 ++++++++++------ python/src/py/objtype.h | 5 + python/src/py/opmethods.c | 4 +- python/src/py/parse.c | 326 ++++++++++++++++++++----------- python/src/py/parsenum.c | 37 +++- python/src/py/persistentcode.c | 6 +- python/src/py/pystack.c | 57 ++++++ python/src/py/pystack.h | 123 ++++++++++++ python/src/py/qstr.c | 35 +--- python/src/py/qstr.h | 4 +- python/src/py/qstrdefs.h | 5 + python/src/py/repl.c | 78 +++----- python/src/py/runtime.c | 125 +++++++----- python/src/py/runtime.h | 9 +- python/src/py/runtime0.h | 4 + python/src/py/runtime_utils.c | 12 +- python/src/py/smallint.h | 6 +- python/src/py/stackctrl.c | 7 +- python/src/py/stream.c | 25 ++- python/src/py/stream.h | 5 +- python/src/py/unicode.c | 29 ++- python/src/py/vm.c | 173 +++++++++------- 100 files changed, 2301 insertions(+), 1417 deletions(-) create mode 100644 python/src/py/emitnarm.c create mode 100644 python/src/py/emitnthumb.c create mode 100644 python/src/py/emitnx64.c create mode 100644 python/src/py/emitnx86.c create mode 100644 python/src/py/emitnxtensa.c create mode 100644 python/src/py/nlr.c create mode 100644 python/src/py/objdeque.c create mode 100644 python/src/py/objnamedtuple.h create mode 100644 python/src/py/pystack.c create mode 100644 python/src/py/pystack.h diff --git a/python/Makefile b/python/Makefile index 7bef56010..ece2546c7 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,34 +1,14 @@ SFLAGS += -Ipython/src SFLAGS += -Ipython/port -# Native code emition is actually not needed -# It's only for the @micropython.native decorator -# which, to be honest, we don't really need - -#BUILD = python/build - -# How to write this makefile -# // Copy PY_O_BASENAME from py.mk and add those to objs -# // Add special-case optimizations - - -# We don't care about emitting multiple archs at once -# so we'll just build one emitnative.o - #emitnx64.o \ - emitnx86.o \ - emitnthumb.o \ - emitnarm.o \ - emitnxtensa.o \ - emitinlinextensa.o \ - - - -# FIXME: Use customized optimization level for some files - - +# How to maintain this Makefile +# - Copy PY_CORE_O_BASENAME from py.mk into py_objs +# - Copy select PY_EXTMOD_O_BASENAME from py.mk into extmod_objs +# - Edit special-case workarounds below as needed py_objs = $(addprefix python/src/py/,\ mpstate.o \ + nlr.o \ nlrx86.o \ nlrx64.o \ nlrthumb.o \ @@ -36,6 +16,7 @@ py_objs = $(addprefix python/src/py/,\ nlrsetjmp.o \ malloc.o \ gc.o \ + pystack.o \ qstr.o \ vstr.o \ mpprint.o \ @@ -50,11 +31,17 @@ py_objs = $(addprefix python/src/py/,\ emitbc.o \ asmbase.o \ asmx64.o \ + emitnx64.o \ asmx86.o \ + emitnx86.o \ asmthumb.o \ + emitnthumb.o \ emitinlinethumb.o \ asmarm.o \ + emitnarm.o \ asmxtensa.o \ + emitnxtensa.o \ + emitinlinextensa.o \ formatfloat.o \ parsenumbase.o \ parsenum.o \ @@ -64,6 +51,7 @@ py_objs = $(addprefix python/src/py/,\ runtime_utils.o \ scheduler.o \ nativeglue.o \ + stackctrl.o \ argcheck.o \ warning.o \ map.o \ @@ -75,6 +63,7 @@ py_objs = $(addprefix python/src/py/,\ objcell.o \ objclosure.o \ objcomplex.o \ + objdeque.o \ objdict.o \ objenumerate.o \ objexcept.o \ @@ -127,58 +116,15 @@ py_objs = $(addprefix python/src/py/,\ vm.o \ bc.o \ showbc.o \ + repl.o \ smallint.o \ frozenmod.o \ ) -# repl.o \ - -py_objs += python/src/py/emitnative.o -python/src/py/emitnative.o: CFLAGS += -DN_THUMB - extmod_objs += $(addprefix python/src/extmod/,\ modurandom.o \ ) -# In order to change the name of the micropython module 'urandom' to 'random' -# (without altering micropython files), we redefined the macro MP_QSTR_urandom -# by DMP_QSTR_random. -python/src/py/objmodule.o: SFLAGS += -DMP_QSTR_urandom="MP_QSTR_random" -python/src/extmod/modurandom.o: SFLAGS += -DMP_QSTR_urandom="MP_QSTR_random" - -#extmod_objs += $(addprefix python/src/extmod/,\ - moductypes.o \ - modujson.o \ - modure.o \ - moduzlib.o \ - moduheapq.o \ - modutimeq.o \ - moduhashlib.o \ - modubinascii.o \ - virtpin.o \ - machine_mem.o \ - machine_pinbase.o \ - machine_signal.o \ - machine_pulse.o \ - machine_i2c.o \ - machine_spi.o \ - modussl_axtls.o \ - modussl_mbedtls.o \ - modurandom.o \ - moduselect.o \ - modwebsocket.o \ - modwebrepl.o \ - modframebuf.o \ - vfs.o \ - vfs_reader.o \ - vfs_fat.o \ - vfs_fat_diskio.o \ - vfs_fat_file.o \ - vfs_fat_misc.o \ - utime_mphal.o \ - uos_dupterm.o \ -) - port_objs += $(addprefix python/port/,\ port.o \ builtins.o\ @@ -186,21 +132,35 @@ port_objs += $(addprefix python/port/,\ modkandinsky.o \ modkandinsky_impl.o \ mphalport.o \ - stackctrl.o \ ) -# Reduce optimization for the emscripten platform. -# With optimization, register and stack variables might be held in a JavaScript -# local variable, which breaks garbage collection. Indeed, these JavaScript +# Workarounds + +# Rename urandom to random +# In order to change the name of the micropython module 'urandom' to 'random' +# (without altering micropython files), we redefined the macro MP_QSTR_urandom +# by DMP_QSTR_random. +python/src/py/objmodule.o: SFLAGS += -DMP_QSTR_urandom="MP_QSTR_random" +python/src/extmod/modurandom.o: SFLAGS += -DMP_QSTR_urandom="MP_QSTR_random" + +# Handle upward-growing stack +# Some platforms such as emscripten have a stack that grows up. We've rewritten +# the stack control file to handle this case. +py_objs := $(filter-out python/src/py/stackctrl.o, $(py_objs)) +port_objs += python/port/stackctrl.o + +# Fix the GC on emscripten +# With optimizations, register and stack variables might be held in a JavaScript +# local variables, which breaks garbage collection. Indeed, these JavaScript # variables cannot be marked as root during garbage collection, which means that # the heap objects they depend on will likely be destroyed. When the Python # computing resumes, if necessary heap objects have been destroyed, the Python # program crashes. - ifeq ($(PLATFORM),emscripten) $(py_objs): SFLAGS := $(subst -Os,-O0,$(SFLAGS)) endif + # QSTR generation generated_headers += $(addprefix python/port/genhdr/, qstrdefs.generated.h) diff --git a/python/port/builtins.c b/python/port/builtins.c index 703831223..a04ad75c6 100644 --- a/python/port/builtins.c +++ b/python/port/builtins.c @@ -19,7 +19,7 @@ mp_obj_t mp_builtin_input(size_t n_args, const mp_obj_t *args) { const char * result = mp_hal_input(prompt); // 3 - Log the prompt, result and flush a new line - mp_obj_t resultStr = mp_obj_new_str(result, strlen(result), false); + mp_obj_t resultStr = mp_obj_new_str(result, strlen(result)); if (n_args == 1) { mp_obj_print(args[0], PRINT_STR); } diff --git a/python/port/genhdr/mpversion.h b/python/port/genhdr/mpversion.h index d99c08687..d3f493cf8 100644 --- a/python/port/genhdr/mpversion.h +++ b/python/port/genhdr/mpversion.h @@ -1,4 +1,4 @@ #define MICROPY_VERSION_MAJOR (1) #define MICROPY_VERSION_MINOR (9) -#define MICROPY_VERSION_MICRO (3) -#define MICROPY_VERSION_STRING "1.9.3" +#define MICROPY_VERSION_MICRO (4) +#define MICROPY_VERSION_STRING "1.9.4" diff --git a/python/port/genhdr/qstrdefs.in.h b/python/port/genhdr/qstrdefs.in.h index 61442cce3..e0b9f738f 100644 --- a/python/port/genhdr/qstrdefs.in.h +++ b/python/port/genhdr/qstrdefs.in.h @@ -1,16 +1,31 @@ -/* In the standard MicroPython build system, this file is generated from the - * other sources and called qstrdefs.preprocessed.h. - * We're bypassing this step our build system, so if you need a QSTR just add it - * below. */ +/* In the standard MicroPython build system, this file is autogenerated from the + * reset of the sources and called qstrdefs.preprocessed.h. We're bypassing this + * step in our build system, so if you need a QSTR just add it below. + * + * How to update this file with a new MicroPython release + * - Get a clean copy of MicroPython + * - Copy our mpconfigport.h over the "bare-arm" port of MicroPython + * - "make" the bare-arm port of MicroPython (don't worry if it doesn't finish) + * - "cat build/genhdr/qstrdefs.preprocessed.h|grep '^Q'|sort|uniq" + * - Insert the result below in the MicroPython QSTRs section + * - remove "QSTR(urandom)" as we renamed it to random */ -QCFG(BYTES_IN_HASH, 1) -QCFG(BYTES_IN_LEN, 1) +// Global configuration +QCFG(BYTES_IN_HASH, (1)) +QCFG(BYTES_IN_LEN, (1)) + +// Kandinsky QSTRs + +Q(kandinsky) Q(color) Q(draw_string) Q(get_pixel) Q(set_pixel) +// MicroPython QSTRs + +Q( ) Q(%#o) Q(%#x) Q() @@ -28,13 +43,9 @@ Q(ArithmeticError) Q(AssertionError) Q(AttributeError) Q(BaseException) -Q(BufferError) Q(EOFError) Q(Ellipsis) Q(Exception) -Q(FileExistsError) -Q(FileNotFoundError) -Q(FloatingPointError) Q(GeneratorExit) Q(ImportError) Q(IndentationError) @@ -53,11 +64,11 @@ Q(StopIteration) Q(SyntaxError) Q(SystemExit) Q(TypeError) -Q(UnboundLocalError) Q(ValueError) Q(ZeroDivisionError) Q(\n) Q(_) +Q(_0x0a_) Q(__add__) Q(__bool__) Q(__build_class__) @@ -80,7 +91,6 @@ Q(__isub__) Q(__iter__) Q(__le__) Q(__len__) -Q(__locals__) Q(__lt__) Q(__main__) Q(__module__) @@ -95,7 +105,7 @@ Q(__setitem__) Q(__str__) Q(__sub__) Q(__traceback__) -Q(__brace_open__colon__hash_b_brace_close_) +Q(_brace_open__colon__hash_b_brace_close_) Q(_lt_dictcomp_gt_) Q(_lt_genexpr_gt_) Q(_lt_lambda_gt_) @@ -105,6 +115,7 @@ Q(_lt_setcomp_gt_) Q(_lt_string_gt_) Q(_percent__hash_o) Q(_percent__hash_x) +Q(_space_) Q(_star_) Q(abs) Q(acos) @@ -126,6 +137,7 @@ Q(bytecode) Q(bytes) Q(callable) Q(ceil) +Q(choice) Q(chr) Q(classmethod) Q(clear) @@ -169,6 +181,7 @@ Q(gamma) Q(generator) Q(get) Q(getattr) +Q(getrandbits) Q(globals) Q(hasattr) Q(hash) @@ -214,6 +227,7 @@ Q(map) Q(math) Q(max) Q(maximum recursion depth exceeded) +Q(maximum_space_recursion_space_depth_space_exceeded) Q(micropython) Q(min) Q(modf) @@ -224,6 +238,7 @@ Q(oct) Q(open) Q(opt_level) Q(ord) +Q(pend_throw) Q(phase) Q(pi) Q(polar) @@ -232,6 +247,9 @@ Q(popitem) Q(pow) Q(print) Q(radians) +Q(randint) +Q(random) +Q(randrange) Q(range) Q(real) Q(rect) @@ -244,6 +262,7 @@ Q(rindex) Q(round) Q(rsplit) Q(rstrip) +Q(seed) Q(send) Q(sep) Q(setattr) @@ -271,6 +290,7 @@ Q(to_bytes) Q(trunc) Q(tuple) Q(type) +Q(uniform) Q(update) Q(upper) Q(utf-8) @@ -278,11 +298,3 @@ Q(value) Q(values) Q(zip) Q({:#b}) -Q(getrandbits) -Q(seed) -Q(random) -Q(randint) -Q(randrange) -Q(randfloat) -Q(choice) -Q(uniform) diff --git a/python/src/extmod/virtpin.h b/python/src/extmod/virtpin.h index 041010350..706affc19 100644 --- a/python/src/extmod/virtpin.h +++ b/python/src/extmod/virtpin.h @@ -23,6 +23,8 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ +#ifndef MICROPY_INCLUDED_EXTMOD_VIRTPIN_H +#define MICROPY_INCLUDED_EXTMOD_VIRTPIN_H #include "py/obj.h" @@ -41,3 +43,5 @@ void mp_virtual_pin_write(mp_obj_t pin, int value); // If a port exposes a Pin object, it's constructor should be like this mp_obj_t mp_pin_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args); + +#endif // MICROPY_INCLUDED_EXTMOD_VIRTPIN_H diff --git a/python/src/py/asmarm.c b/python/src/py/asmarm.c index 552fdfb34..1a8923bc2 100644 --- a/python/src/py/asmarm.c +++ b/python/src/py/asmarm.c @@ -150,10 +150,7 @@ void asm_arm_bkpt(asm_arm_t *as) { // | low address | high address in RAM void asm_arm_entry(asm_arm_t *as, int num_locals) { - - if (num_locals < 0) { - num_locals = 0; - } + assert(num_locals >= 0); as->stack_adjust = 0; as->push_reglist = 1 << ASM_ARM_REG_R1 diff --git a/python/src/py/asmarm.h b/python/src/py/asmarm.h index a302b1590..871e35820 100644 --- a/python/src/py/asmarm.h +++ b/python/src/py/asmarm.h @@ -167,17 +167,12 @@ void asm_arm_bl_ind(asm_arm_t *as, void *fun_ptr, uint fun_id, uint reg_temp); } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_arm_bl_ind(as, ptr, idx, ASM_ARM_REG_R3) -#define ASM_MOV_REG_TO_LOCAL(as, reg, local_num) asm_arm_mov_local_reg(as, (local_num), (reg)) -#define ASM_MOV_IMM_TO_REG(as, imm, reg) asm_arm_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_ALIGNED_IMM_TO_REG(as, imm, reg) asm_arm_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_arm_mov_reg_i32(as, (reg_temp), (imm)); \ - asm_arm_mov_local_reg(as, (local_num), (reg_temp)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG(as, local_num, reg) asm_arm_mov_reg_local(as, (reg), (local_num)) +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_arm_mov_local_reg((as), (local_num), (reg_src)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_arm_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_arm_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_arm_mov_reg_reg((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG(as, local_num, reg) asm_arm_mov_reg_local_addr(as, (reg), (local_num)) +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_arm_mov_reg_local_addr((as), (reg_dest), (local_num)) #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) asm_arm_lsl_reg_reg((as), (reg_dest), (reg_shift)) #define ASM_ASR_REG_REG(as, reg_dest, reg_shift) asm_arm_asr_reg_reg((as), (reg_dest), (reg_shift)) diff --git a/python/src/py/asmbase.c b/python/src/py/asmbase.c index c941e917b..4c84c3b25 100644 --- a/python/src/py/asmbase.c +++ b/python/src/py/asmbase.c @@ -46,10 +46,10 @@ void mp_asm_base_deinit(mp_asm_base_t *as, bool free_code) { } void mp_asm_base_start_pass(mp_asm_base_t *as, int pass) { - if (pass == MP_ASM_PASS_COMPUTE) { - // reset all labels + if (pass < MP_ASM_PASS_EMIT) { + // Reset labels so we can detect backwards jumps (and verify unique assignment) memset(as->label_offsets, -1, as->max_num_labels * sizeof(size_t)); - } else if (pass == MP_ASM_PASS_EMIT) { + } else { // allocating executable RAM is platform specific MP_PLAT_ALLOC_EXEC(as->code_offset, (void**)&as->code_base, &as->code_size); assert(as->code_base != NULL); diff --git a/python/src/py/asmthumb.c b/python/src/py/asmthumb.c index 5316a7efb..c5b45f2f5 100644 --- a/python/src/py/asmthumb.c +++ b/python/src/py/asmthumb.c @@ -104,6 +104,8 @@ STATIC void asm_thumb_write_word32(asm_thumb_t *as, int w32) { // | low address | high address in RAM void asm_thumb_entry(asm_thumb_t *as, int num_locals) { + assert(num_locals >= 0); + // work out what to push and how many extra spaces to reserve on stack // so that we have enough for all locals and it's aligned an 8-byte boundary // we push extra regs (r1, r2, r3) to help do the stack adjustment @@ -111,9 +113,6 @@ void asm_thumb_entry(asm_thumb_t *as, int num_locals) { // for push rlist, lowest numbered register at the lowest address uint reglist; uint stack_adjust; - if (num_locals < 0) { - num_locals = 0; - } // don't pop r0 because it's used for return value switch (num_locals) { case 0: diff --git a/python/src/py/asmthumb.h b/python/src/py/asmthumb.h index 7070e03ac..8a7df5d50 100644 --- a/python/src/py/asmthumb.h +++ b/python/src/py/asmthumb.h @@ -26,6 +26,7 @@ #ifndef MICROPY_INCLUDED_PY_ASMTHUMB_H #define MICROPY_INCLUDED_PY_ASMTHUMB_H +#include #include "py/misc.h" #include "py/asmbase.h" @@ -283,17 +284,12 @@ void asm_thumb_bl_ind(asm_thumb_t *as, void *fun_ptr, uint fun_id, uint reg_temp } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_thumb_bl_ind(as, ptr, idx, ASM_THUMB_REG_R3) -#define ASM_MOV_REG_TO_LOCAL(as, reg, local_num) asm_thumb_mov_local_reg(as, (local_num), (reg)) -#define ASM_MOV_IMM_TO_REG(as, imm, reg) asm_thumb_mov_reg_i32_optimised(as, (reg), (imm)) -#define ASM_MOV_ALIGNED_IMM_TO_REG(as, imm, reg) asm_thumb_mov_reg_i32_aligned(as, (reg), (imm)) -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_thumb_mov_reg_i32_optimised(as, (reg_temp), (imm)); \ - asm_thumb_mov_local_reg(as, (local_num), (reg_temp)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG(as, local_num, reg) asm_thumb_mov_reg_local(as, (reg), (local_num)) +#define ASM_MOV_LOCAL_REG(as, local_num, reg) asm_thumb_mov_local_reg((as), (local_num), (reg)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_optimised((as), (reg_dest), (imm)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_thumb_mov_reg_i32_aligned((as), (reg_dest), (imm)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_thumb_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_thumb_mov_reg_reg((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG(as, local_num, reg) asm_thumb_mov_reg_local_addr(as, (reg), (local_num)) +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_thumb_mov_reg_local_addr((as), (reg_dest), (local_num)) #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) asm_thumb_format_4((as), ASM_THUMB_FORMAT_4_LSL, (reg_dest), (reg_shift)) #define ASM_ASR_REG_REG(as, reg_dest, reg_shift) asm_thumb_format_4((as), ASM_THUMB_FORMAT_4_ASR, (reg_dest), (reg_shift)) diff --git a/python/src/py/asmx64.c b/python/src/py/asmx64.c index aa2a8ec7c..c900a08d1 100644 --- a/python/src/py/asmx64.c +++ b/python/src/py/asmx64.c @@ -526,11 +526,9 @@ void asm_x64_jcc_label(asm_x64_t *as, int jcc_type, mp_uint_t label) { } void asm_x64_entry(asm_x64_t *as, int num_locals) { + assert(num_locals >= 0); asm_x64_push_r64(as, ASM_X64_REG_RBP); asm_x64_mov_r64_r64(as, ASM_X64_REG_RBP, ASM_X64_REG_RSP); - if (num_locals < 0) { - num_locals = 0; - } num_locals |= 1; // make it odd so stack is aligned on 16 byte boundary asm_x64_sub_r64_i32(as, ASM_X64_REG_RSP, num_locals * WORD_SIZE); asm_x64_push_r64(as, ASM_X64_REG_RBX); diff --git a/python/src/py/asmx64.h b/python/src/py/asmx64.h index 425bdf2d3..2fbbfa9ff 100644 --- a/python/src/py/asmx64.h +++ b/python/src/py/asmx64.h @@ -162,17 +162,12 @@ void asm_x64_call_ind(asm_x64_t* as, void* ptr, int temp_r32); } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_x64_call_ind(as, ptr, ASM_X64_REG_RAX) -#define ASM_MOV_REG_TO_LOCAL asm_x64_mov_r64_to_local -#define ASM_MOV_IMM_TO_REG asm_x64_mov_i64_to_r64_optimised -#define ASM_MOV_ALIGNED_IMM_TO_REG asm_x64_mov_i64_to_r64_aligned -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_x64_mov_i64_to_r64_optimised(as, (imm), (reg_temp)); \ - asm_x64_mov_r64_to_local(as, (reg_temp), (local_num)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG asm_x64_mov_local_to_r64 +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x64_mov_r64_to_local((as), (reg_src), (local_num)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_optimised((as), (imm), (reg_dest)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_x64_mov_i64_to_r64_aligned((as), (imm), (reg_dest)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x64_mov_local_to_r64((as), (local_num), (reg_dest)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x64_mov_r64_r64((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG asm_x64_mov_local_addr_to_r64 +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x64_mov_local_addr_to_r64((as), (local_num), (reg_dest)) #define ASM_LSL_REG(as, reg) asm_x64_shl_r64_cl((as), (reg)) #define ASM_ASR_REG(as, reg) asm_x64_sar_r64_cl((as), (reg)) diff --git a/python/src/py/asmx86.c b/python/src/py/asmx86.c index 6a78fbd5e..3938baaac 100644 --- a/python/src/py/asmx86.c +++ b/python/src/py/asmx86.c @@ -387,7 +387,8 @@ void asm_x86_jcc_label(asm_x86_t *as, mp_uint_t jcc_type, mp_uint_t label) { } } -void asm_x86_entry(asm_x86_t *as, mp_uint_t num_locals) { +void asm_x86_entry(asm_x86_t *as, int num_locals) { + assert(num_locals >= 0); asm_x86_push_r32(as, ASM_X86_REG_EBP); asm_x86_mov_r32_r32(as, ASM_X86_REG_EBP, ASM_X86_REG_ESP); if (num_locals > 0) { diff --git a/python/src/py/asmx86.h b/python/src/py/asmx86.h index 0a00e2e7c..09559850c 100644 --- a/python/src/py/asmx86.h +++ b/python/src/py/asmx86.h @@ -104,7 +104,7 @@ void asm_x86_test_r8_with_r8(asm_x86_t* as, int src_r32_a, int src_r32_b); void asm_x86_setcc_r8(asm_x86_t* as, mp_uint_t jcc_type, int dest_r8); void asm_x86_jmp_label(asm_x86_t* as, mp_uint_t label); void asm_x86_jcc_label(asm_x86_t* as, mp_uint_t jcc_type, mp_uint_t label); -void asm_x86_entry(asm_x86_t* as, mp_uint_t num_locals); +void asm_x86_entry(asm_x86_t* as, int num_locals); void asm_x86_exit(asm_x86_t* as); void asm_x86_mov_arg_to_r32(asm_x86_t *as, int src_arg_num, int dest_r32); void asm_x86_mov_local_to_r32(asm_x86_t* as, int src_local_num, int dest_r32); @@ -160,17 +160,12 @@ void asm_x86_call_ind(asm_x86_t* as, void* ptr, mp_uint_t n_args, int temp_r32); } while (0) #define ASM_CALL_IND(as, ptr, idx) asm_x86_call_ind(as, ptr, mp_f_n_args[idx], ASM_X86_REG_EAX) -#define ASM_MOV_REG_TO_LOCAL asm_x86_mov_r32_to_local -#define ASM_MOV_IMM_TO_REG asm_x86_mov_i32_to_r32 -#define ASM_MOV_ALIGNED_IMM_TO_REG asm_x86_mov_i32_to_r32_aligned -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_x86_mov_i32_to_r32(as, (imm), (reg_temp)); \ - asm_x86_mov_r32_to_local(as, (reg_temp), (local_num)); \ - } while (false) -#define ASM_MOV_LOCAL_TO_REG asm_x86_mov_local_to_r32 +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_x86_mov_r32_to_local((as), (reg_src), (local_num)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32((as), (imm), (reg_dest)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_x86_mov_i32_to_r32_aligned((as), (imm), (reg_dest)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_x86_mov_local_to_r32((as), (local_num), (reg_dest)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_x86_mov_r32_r32((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG asm_x86_mov_local_addr_to_r32 +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_x86_mov_local_addr_to_r32((as), (local_num), (reg_dest)) #define ASM_LSL_REG(as, reg) asm_x86_shl_r32_cl((as), (reg)) #define ASM_ASR_REG(as, reg) asm_x86_sar_r32_cl((as), (reg)) diff --git a/python/src/py/asmxtensa.h b/python/src/py/asmxtensa.h index 7db6c0d3d..e6d4158cb 100644 --- a/python/src/py/asmxtensa.h +++ b/python/src/py/asmxtensa.h @@ -280,17 +280,12 @@ void asm_xtensa_mov_reg_local_addr(asm_xtensa_t *as, uint reg_dest, int local_nu asm_xtensa_op_callx0(as, ASM_XTENSA_REG_A0); \ } while (0) -#define ASM_MOV_REG_TO_LOCAL(as, reg, local_num) asm_xtensa_mov_local_reg(as, (local_num), (reg)) -#define ASM_MOV_IMM_TO_REG(as, imm, reg) asm_xtensa_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_ALIGNED_IMM_TO_REG(as, imm, reg) asm_xtensa_mov_reg_i32(as, (reg), (imm)) -#define ASM_MOV_IMM_TO_LOCAL_USING(as, imm, local_num, reg_temp) \ - do { \ - asm_xtensa_mov_reg_i32(as, (reg_temp), (imm)); \ - asm_xtensa_mov_local_reg(as, (local_num), (reg_temp)); \ - } while (0) -#define ASM_MOV_LOCAL_TO_REG(as, local_num, reg) asm_xtensa_mov_reg_local(as, (reg), (local_num)) +#define ASM_MOV_LOCAL_REG(as, local_num, reg_src) asm_xtensa_mov_local_reg((as), (local_num), (reg_src)) +#define ASM_MOV_REG_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_ALIGNED_IMM(as, reg_dest, imm) asm_xtensa_mov_reg_i32((as), (reg_dest), (imm)) +#define ASM_MOV_REG_LOCAL(as, reg_dest, local_num) asm_xtensa_mov_reg_local((as), (reg_dest), (local_num)) #define ASM_MOV_REG_REG(as, reg_dest, reg_src) asm_xtensa_op_mov_n((as), (reg_dest), (reg_src)) -#define ASM_MOV_LOCAL_ADDR_TO_REG(as, local_num, reg) asm_xtensa_mov_reg_local_addr(as, (reg), (local_num)) +#define ASM_MOV_REG_LOCAL_ADDR(as, reg_dest, local_num) asm_xtensa_mov_reg_local_addr((as), (reg_dest), (local_num)) #define ASM_LSL_REG_REG(as, reg_dest, reg_shift) \ do { \ diff --git a/python/src/py/binary.c b/python/src/py/binary.c index 870a0942b..f509ff010 100644 --- a/python/src/py/binary.c +++ b/python/src/py/binary.c @@ -206,7 +206,7 @@ mp_obj_t mp_binary_get_val(char struct_type, char val_type, byte **ptr) { return (mp_obj_t)(mp_uint_t)val; } else if (val_type == 'S') { const char *s_val = (const char*)(uintptr_t)(mp_uint_t)val; - return mp_obj_new_str(s_val, strlen(s_val), false); + return mp_obj_new_str(s_val, strlen(s_val)); #if MICROPY_PY_BUILTINS_FLOAT } else if (val_type == 'f') { union { uint32_t i; float f; } fpu = {val}; diff --git a/python/src/py/builtinhelp.c b/python/src/py/builtinhelp.c index c9992906d..6c2c3b92c 100644 --- a/python/src/py/builtinhelp.c +++ b/python/src/py/builtinhelp.c @@ -69,7 +69,7 @@ STATIC void mp_help_add_from_names(mp_obj_t list, const char *name) { while (*name) { size_t l = strlen(name); // name should end in '.py' and we strip it off - mp_obj_list_append(list, mp_obj_new_str(name, l - 3, false)); + mp_obj_list_append(list, mp_obj_new_str(name, l - 3)); name += l + 1; } } @@ -100,7 +100,7 @@ STATIC void mp_help_print_modules(void) { // print the list of modules in a column-first order #define NUM_COLUMNS (4) #define COLUMN_WIDTH (18) - mp_uint_t len; + size_t len; mp_obj_t *items; mp_obj_list_get(list, &len, &items); unsigned int num_rows = (len + NUM_COLUMNS - 1) / NUM_COLUMNS; diff --git a/python/src/py/builtinimport.c b/python/src/py/builtinimport.c index 04ce66723..b8ed096ca 100644 --- a/python/src/py/builtinimport.c +++ b/python/src/py/builtinimport.c @@ -44,6 +44,8 @@ #define DEBUG_printf(...) (void)0 #endif +#if MICROPY_ENABLE_EXTERNAL_IMPORT + #define PATH_SEP_CHAR '/' bool mp_obj_is_package(mp_obj_t module) { @@ -318,7 +320,7 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } uint new_mod_l = (mod_len == 0 ? (size_t)(p - this_name) : (size_t)(p - this_name) + 1 + mod_len); - char *new_mod = alloca(new_mod_l); + char *new_mod = mp_local_alloc(new_mod_l); memcpy(new_mod, this_name, p - this_name); if (mod_len != 0) { new_mod[p - this_name] = '.'; @@ -326,9 +328,10 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } qstr new_mod_q = qstr_from_strn(new_mod, new_mod_l); + mp_local_free(new_mod); DEBUG_printf("Resolved base name for relative import: '%s'\n", qstr_str(new_mod_q)); module_name = MP_OBJ_NEW_QSTR(new_mod_q); - mod_str = new_mod; + mod_str = qstr_str(new_mod_q); mod_len = new_mod_l; } @@ -388,6 +391,7 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { } // found weak linked module module_obj = el->value; + mp_module_call_init(mod_name, module_obj); } else { no_exist: #else @@ -434,7 +438,7 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { DEBUG_printf("%.*s is dir\n", vstr_len(&path), vstr_str(&path)); // https://docs.python.org/3/reference/import.html // "Specifically, any module that contains a __path__ attribute is considered a package." - mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(&path), vstr_len(&path), false)); + mp_store_attr(module_obj, MP_QSTR___path__, mp_obj_new_str(vstr_str(&path), vstr_len(&path))); size_t orig_path_len = path.len; vstr_add_char(&path, PATH_SEP_CHAR); vstr_add_str(&path, "__init__.py"); @@ -471,4 +475,41 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { // Otherwise, we need to return top-level package return top_module_obj; } + +#else // MICROPY_ENABLE_EXTERNAL_IMPORT + +mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) { + // Check that it's not a relative import + if (n_args >= 5 && MP_OBJ_SMALL_INT_VALUE(args[4]) != 0) { + mp_raise_NotImplementedError("relative import"); + } + + // Check if module already exists, and return it if it does + qstr module_name_qstr = mp_obj_str_get_qstr(args[0]); + mp_obj_t module_obj = mp_module_get(module_name_qstr); + if (module_obj != MP_OBJ_NULL) { + return module_obj; + } + + #if MICROPY_MODULE_WEAK_LINKS + // Check if there is a weak link to this module + mp_map_elem_t *el = mp_map_lookup((mp_map_t*)&mp_builtin_module_weak_links_map, MP_OBJ_NEW_QSTR(module_name_qstr), MP_MAP_LOOKUP); + if (el != NULL) { + // Found weak-linked module + mp_module_call_init(module_name_qstr, el->value); + return el->value; + } + #endif + + // Couldn't find the module, so fail + if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { + mp_raise_msg(&mp_type_ImportError, "module not found"); + } else { + nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ImportError, + "no module named '%q'", module_name_qstr)); + } +} + +#endif // MICROPY_ENABLE_EXTERNAL_IMPORT + MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin___import___obj, 1, 5, mp_builtin___import__); diff --git a/python/src/py/compile.c b/python/src/py/compile.c index 4e704abfb..9200b346b 100644 --- a/python/src/py/compile.c +++ b/python/src/py/compile.c @@ -376,6 +376,7 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as EMIT(store_subscr); } } + return; } else if (MP_PARSE_NODE_STRUCT_KIND(pns1) == PN_trailer_period) { assert(MP_PARSE_NODE_IS_ID(pns1->nodes[0])); if (assign_kind == ASSIGN_AUG_LOAD) { @@ -387,16 +388,10 @@ STATIC void c_assign_atom_expr(compiler_t *comp, mp_parse_node_struct_t *pns, as } EMIT_ARG(store_attr, MP_PARSE_NODE_LEAF_ARG(pns1->nodes[0])); } - } else { - goto cannot_assign; + return; } - } else { - goto cannot_assign; } - return; - -cannot_assign: compile_syntax_error(comp, (mp_parse_node_t)pns, "can't assign to expression"); } @@ -1050,8 +1045,8 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { for (int i = 0; i < n; i++) { len += qstr_len(MP_PARSE_NODE_LEAF_ARG(pns->nodes[i])); } - byte *q_ptr; - byte *str_dest = qstr_build_start(len, &q_ptr); + char *q_ptr = mp_local_alloc(len); + char *str_dest = q_ptr; for (int i = 0; i < n; i++) { if (i > 0) { *str_dest++ = '.'; @@ -1061,7 +1056,8 @@ STATIC void do_import_name(compiler_t *comp, mp_parse_node_t pn, qstr *q_base) { memcpy(str_dest, str_src, str_src_len); str_dest += str_src_len; } - qstr q_full = qstr_build_end(q_ptr); + qstr q_full = qstr_from_strn(q_ptr, len); + mp_local_free(q_ptr); EMIT_ARG(import_name, q_full); if (is_as) { for (int i = 1; i < n; i++) { @@ -1940,51 +1936,52 @@ STATIC void compile_expr_stmt(compiler_t *comp, mp_parse_node_struct_t *pns) { } } else { plain_assign: - if (MICROPY_COMP_DOUBLE_TUPLE_ASSIGN - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_testlist_star_expr) - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[1]) == 2 - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 2) { - // optimisation for a, b = c, d - mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns->nodes[1]; + #if MICROPY_COMP_DOUBLE_TUPLE_ASSIGN + if (MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_testlist_star_expr) + && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr)) { mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; - if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) - || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr)) { - // can't optimise when it's a star expression on the lhs - goto no_optimisation; + pns1 = (mp_parse_node_struct_t*)pns->nodes[1]; + uint32_t n_pns0 = MP_PARSE_NODE_STRUCT_NUM_NODES(pns0); + // Can only optimise a tuple-to-tuple assignment when all of the following hold: + // - equal number of items in LHS and RHS tuples + // - 2 or 3 items in the tuples + // - there are no star expressions in the LHS tuple + if (n_pns0 == MP_PARSE_NODE_STRUCT_NUM_NODES(pns1) + && (n_pns0 == 2 + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + || n_pns0 == 3 + #endif + ) + && !MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) + && !MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr) + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + && (n_pns0 == 2 || !MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[2], PN_star_expr)) + #endif + ) { + // Optimisation for a, b = c, d or a, b, c = d, e, f + compile_node(comp, pns1->nodes[0]); // rhs + compile_node(comp, pns1->nodes[1]); // rhs + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + if (n_pns0 == 3) { + compile_node(comp, pns1->nodes[2]); // rhs + EMIT(rot_three); + } + #endif + EMIT(rot_two); + c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store + c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store + #if MICROPY_COMP_TRIPLE_TUPLE_ASSIGN + if (n_pns0 == 3) { + c_assign(comp, pns0->nodes[2], ASSIGN_STORE); // lhs store + } + #endif + return; } - compile_node(comp, pns10->nodes[0]); // rhs - compile_node(comp, pns10->nodes[1]); // rhs - EMIT(rot_two); - c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store - c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store - } else if (MICROPY_COMP_TRIPLE_TUPLE_ASSIGN - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[1], PN_testlist_star_expr) - && MP_PARSE_NODE_IS_STRUCT_KIND(pns->nodes[0], PN_testlist_star_expr) - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[1]) == 3 - && MP_PARSE_NODE_STRUCT_NUM_NODES((mp_parse_node_struct_t*)pns->nodes[0]) == 3) { - // optimisation for a, b, c = d, e, f - mp_parse_node_struct_t *pns10 = (mp_parse_node_struct_t*)pns->nodes[1]; - mp_parse_node_struct_t *pns0 = (mp_parse_node_struct_t*)pns->nodes[0]; - if (MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[0], PN_star_expr) - || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[1], PN_star_expr) - || MP_PARSE_NODE_IS_STRUCT_KIND(pns0->nodes[2], PN_star_expr)) { - // can't optimise when it's a star expression on the lhs - goto no_optimisation; - } - compile_node(comp, pns10->nodes[0]); // rhs - compile_node(comp, pns10->nodes[1]); // rhs - compile_node(comp, pns10->nodes[2]); // rhs - EMIT(rot_three); - EMIT(rot_two); - c_assign(comp, pns0->nodes[0], ASSIGN_STORE); // lhs store - c_assign(comp, pns0->nodes[1], ASSIGN_STORE); // lhs store - c_assign(comp, pns0->nodes[2], ASSIGN_STORE); // lhs store - } else { - no_optimisation: - compile_node(comp, pns->nodes[1]); // rhs - c_assign(comp, pns->nodes[0], ASSIGN_STORE); // lhs store } + #endif + + compile_node(comp, pns->nodes[1]); // rhs + c_assign(comp, pns->nodes[0], ASSIGN_STORE); // lhs store } } else { goto plain_assign; diff --git a/python/src/py/emitbc.c b/python/src/py/emitbc.c index 3f4dfc178..32e833000 100644 --- a/python/src/py/emitbc.c +++ b/python/src/py/emitbc.c @@ -313,9 +313,12 @@ void mp_emit_bc_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scope) { emit->scope = scope; emit->last_source_line_offset = 0; emit->last_source_line = 1; + #ifndef NDEBUG + // With debugging enabled labels are checked for unique assignment if (pass < MP_PASS_EMIT) { memset(emit->label_offsets, -1, emit->max_num_labels * sizeof(mp_uint_t)); } + #endif emit->bytecode_offset = 0; emit->code_info_offset = 0; @@ -434,7 +437,9 @@ void mp_emit_bc_end_pass(emit_t *emit) { } else if (emit->pass == MP_PASS_EMIT) { mp_emit_glue_assign_bytecode(emit->scope->raw_code, emit->code_base, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS emit->code_info_size + emit->bytecode_size, + #endif emit->const_table, #if MICROPY_PERSISTENT_CODE_SAVE emit->ct_cur_obj, emit->ct_cur_raw_code, @@ -495,7 +500,6 @@ void mp_emit_bc_label_assign(emit_t *emit, mp_uint_t l) { emit->label_offsets[l] = emit->bytecode_offset; } else { // ensure label offset has not changed from MP_PASS_CODE_SIZE to MP_PASS_EMIT - //printf("l%d: (at %d vs %d)\n", l, emit->bytecode_offset, emit->label_offsets[l]); assert(emit->label_offsets[l] == emit->bytecode_offset); } } diff --git a/python/src/py/emitglue.c b/python/src/py/emitglue.c index d2add988f..74bf8ddca 100644 --- a/python/src/py/emitglue.c +++ b/python/src/py/emitglue.c @@ -55,7 +55,10 @@ mp_raw_code_t *mp_emit_glue_new_raw_code(void) { return rc; } -void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, mp_uint_t len, +void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t len, + #endif const mp_uint_t *const_table, #if MICROPY_PERSISTENT_CODE_SAVE uint16_t n_obj, uint16_t n_raw_code, diff --git a/python/src/py/emitglue.h b/python/src/py/emitglue.h index 43930333d..0830a0d5c 100644 --- a/python/src/py/emitglue.h +++ b/python/src/py/emitglue.h @@ -40,7 +40,7 @@ typedef enum { } mp_raw_code_kind_t; typedef struct _mp_raw_code_t { - mp_raw_code_kind_t kind : 3; + mp_uint_t kind : 3; // of type mp_raw_code_kind_t mp_uint_t scope_flags : 7; mp_uint_t n_pos_args : 11; union { @@ -63,7 +63,10 @@ typedef struct _mp_raw_code_t { mp_raw_code_t *mp_emit_glue_new_raw_code(void); -void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, mp_uint_t len, +void mp_emit_glue_assign_bytecode(mp_raw_code_t *rc, const byte *code, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + size_t len, + #endif const mp_uint_t *const_table, #if MICROPY_PERSISTENT_CODE_SAVE uint16_t n_obj, uint16_t n_raw_code, diff --git a/python/src/py/emitnarm.c b/python/src/py/emitnarm.c new file mode 100644 index 000000000..1b585f821 --- /dev/null +++ b/python/src/py/emitnarm.c @@ -0,0 +1,15 @@ +// ARM specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_ARM + +// This is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmarm.h" + +#define N_ARM (1) +#define EXPORT_FUN(name) emit_native_arm_##name +#include "py/emitnative.c" + +#endif diff --git a/python/src/py/emitnative.c b/python/src/py/emitnative.c index 8e97dda11..7e017ba39 100644 --- a/python/src/py/emitnative.c +++ b/python/src/py/emitnative.c @@ -57,102 +57,14 @@ #endif // wrapper around everything in this file -#if (MICROPY_EMIT_X64 && N_X64) \ - || (MICROPY_EMIT_X86 && N_X86) \ - || (MICROPY_EMIT_THUMB && N_THUMB) \ - || (MICROPY_EMIT_ARM && N_ARM) \ - || (MICROPY_EMIT_XTENSA && N_XTENSA) \ +#if N_X64 || N_X86 || N_THUMB || N_ARM || N_XTENSA -// this is defined so that the assembler exports generic assembler API macros -#define GENERIC_ASM_API (1) - -#if N_X64 - -// x64 specific stuff -#include "py/asmx64.h" -#define EXPORT_FUN(name) emit_native_x64_##name - -#elif N_X86 - -// x86 specific stuff - -STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { - [MP_F_CONVERT_OBJ_TO_NATIVE] = 2, - [MP_F_CONVERT_NATIVE_TO_OBJ] = 2, - [MP_F_LOAD_NAME] = 1, - [MP_F_LOAD_GLOBAL] = 1, - [MP_F_LOAD_BUILD_CLASS] = 0, - [MP_F_LOAD_ATTR] = 2, - [MP_F_LOAD_METHOD] = 3, - [MP_F_LOAD_SUPER_METHOD] = 2, - [MP_F_STORE_NAME] = 2, - [MP_F_STORE_GLOBAL] = 2, - [MP_F_STORE_ATTR] = 3, - [MP_F_OBJ_SUBSCR] = 3, - [MP_F_OBJ_IS_TRUE] = 1, - [MP_F_UNARY_OP] = 2, - [MP_F_BINARY_OP] = 3, - [MP_F_BUILD_TUPLE] = 2, - [MP_F_BUILD_LIST] = 2, - [MP_F_LIST_APPEND] = 2, - [MP_F_BUILD_MAP] = 1, - [MP_F_STORE_MAP] = 3, -#if MICROPY_PY_BUILTINS_SET - [MP_F_BUILD_SET] = 2, - [MP_F_STORE_SET] = 2, -#endif - [MP_F_MAKE_FUNCTION_FROM_RAW_CODE] = 3, - [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3, - [MP_F_CALL_METHOD_N_KW] = 3, - [MP_F_CALL_METHOD_N_KW_VAR] = 3, - [MP_F_NATIVE_GETITER] = 2, - [MP_F_NATIVE_ITERNEXT] = 1, - [MP_F_NLR_PUSH] = 1, - [MP_F_NLR_POP] = 0, - [MP_F_NATIVE_RAISE] = 1, - [MP_F_IMPORT_NAME] = 3, - [MP_F_IMPORT_FROM] = 2, - [MP_F_IMPORT_ALL] = 1, -#if MICROPY_PY_BUILTINS_SLICE - [MP_F_NEW_SLICE] = 3, -#endif - [MP_F_UNPACK_SEQUENCE] = 3, - [MP_F_UNPACK_EX] = 3, - [MP_F_DELETE_NAME] = 1, - [MP_F_DELETE_GLOBAL] = 1, - [MP_F_NEW_CELL] = 1, - [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, - [MP_F_SETUP_CODE_STATE] = 5, - [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, - [MP_F_SMALL_INT_MODULO] = 2, -}; - -#include "py/asmx86.h" -#define EXPORT_FUN(name) emit_native_x86_##name - -#elif N_THUMB - -// thumb specific stuff -#include "py/asmthumb.h" -#define EXPORT_FUN(name) emit_native_thumb_##name - -#elif N_ARM - -// ARM specific stuff -#include "py/asmarm.h" -#define EXPORT_FUN(name) emit_native_arm_##name - -#elif N_XTENSA - -// Xtensa specific stuff -#include "py/asmxtensa.h" -#define EXPORT_FUN(name) emit_native_xtensa_##name - -#else - -#error unknown native emitter - -#endif +// define additional generic helper macros +#define ASM_MOV_LOCAL_IMM_VIA(as, local_num, imm, reg_temp) \ + do { \ + ASM_MOV_REG_IMM((as), (reg_temp), (imm)); \ + ASM_MOV_LOCAL_REG((as), (local_num), (reg_temp)); \ + } while (false) #define EMIT_NATIVE_VIPER_TYPE_ERROR(emit, ...) do { \ *emit->error_slot = mp_obj_new_exception_msg_varg(&mp_type_ViperTypeError, __VA_ARGS__); \ @@ -389,7 +301,7 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop ASM_MOV_REG_REG(emit->as, REG_LOCAL_3, REG_ARG_3); } else { assert(i == 3); // should be true; max 4 args is checked above - ASM_MOV_REG_TO_LOCAL(emit->as, REG_ARG_4, i - REG_LOCAL_NUM); + ASM_MOV_LOCAL_REG(emit->as, i - REG_LOCAL_NUM, REG_ARG_4); } } #endif @@ -418,14 +330,14 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop #endif // set code_state.fun_bc - ASM_MOV_REG_TO_LOCAL(emit->as, REG_ARG_1, offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t)); + ASM_MOV_LOCAL_REG(emit->as, offsetof(mp_code_state_t, fun_bc) / sizeof(uintptr_t), REG_ARG_1); // set code_state.ip (offset from start of this function to prelude info) // XXX this encoding may change size - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, emit->prelude_offset, offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), REG_ARG_1); + ASM_MOV_LOCAL_IMM_VIA(emit->as, offsetof(mp_code_state_t, ip) / sizeof(uintptr_t), emit->prelude_offset, REG_ARG_1); // put address of code_state into first arg - ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, 0, REG_ARG_1); + ASM_MOV_REG_LOCAL_ADDR(emit->as, REG_ARG_1, 0); // call mp_setup_code_state to prepare code_state structure #if N_THUMB @@ -438,11 +350,11 @@ STATIC void emit_native_start_pass(emit_t *emit, pass_kind_t pass, scope_t *scop // cache some locals in registers if (scope->num_locals > 0) { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - 0, REG_LOCAL_1); + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_1, STATE_START + emit->n_state - 1 - 0); if (scope->num_locals > 1) { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - 1, REG_LOCAL_2); + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_2, STATE_START + emit->n_state - 1 - 1); if (scope->num_locals > 2) { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - 2, REG_LOCAL_3); + ASM_MOV_REG_LOCAL(emit->as, REG_LOCAL_3, STATE_START + emit->n_state - 1 - 2); } } } @@ -606,7 +518,7 @@ STATIC void need_reg_single(emit_t *emit, int reg_needed, int skip_stack_pos) { stack_info_t *si = &emit->stack_info[i]; if (si->kind == STACK_REG && si->data.u_reg == reg_needed) { si->kind = STACK_VALUE; - ASM_MOV_REG_TO_LOCAL(emit->as, si->data.u_reg, emit->stack_start + i); + ASM_MOV_LOCAL_REG(emit->as, emit->stack_start + i, si->data.u_reg); } } } @@ -617,7 +529,7 @@ STATIC void need_reg_all(emit_t *emit) { stack_info_t *si = &emit->stack_info[i]; if (si->kind == STACK_REG) { si->kind = STACK_VALUE; - ASM_MOV_REG_TO_LOCAL(emit->as, si->data.u_reg, emit->stack_start + i); + ASM_MOV_LOCAL_REG(emit->as, emit->stack_start + i, si->data.u_reg); } } } @@ -629,7 +541,7 @@ STATIC void need_stack_settled(emit_t *emit) { if (si->kind == STACK_REG) { DEBUG_printf(" reg(%u) to local(%u)\n", si->data.u_reg, emit->stack_start + i); si->kind = STACK_VALUE; - ASM_MOV_REG_TO_LOCAL(emit->as, si->data.u_reg, emit->stack_start + i); + ASM_MOV_LOCAL_REG(emit->as, emit->stack_start + i, si->data.u_reg); } } for (int i = 0; i < emit->stack_size; i++) { @@ -637,7 +549,7 @@ STATIC void need_stack_settled(emit_t *emit) { if (si->kind == STACK_IMM) { DEBUG_printf(" imm(" INT_FMT ") to local(%u)\n", si->data.u_imm, emit->stack_start + i); si->kind = STACK_VALUE; - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, si->data.u_imm, emit->stack_start + i, REG_TEMP0); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + i, si->data.u_imm, REG_TEMP0); } } } @@ -649,7 +561,7 @@ STATIC void emit_access_stack(emit_t *emit, int pos, vtype_kind_t *vtype, int re *vtype = si->vtype; switch (si->kind) { case STACK_VALUE: - ASM_MOV_LOCAL_TO_REG(emit->as, emit->stack_start + emit->stack_size - pos, reg_dest); + ASM_MOV_REG_LOCAL(emit->as, reg_dest, emit->stack_start + emit->stack_size - pos); break; case STACK_REG: @@ -659,7 +571,7 @@ STATIC void emit_access_stack(emit_t *emit, int pos, vtype_kind_t *vtype, int re break; case STACK_IMM: - ASM_MOV_IMM_TO_REG(emit->as, si->data.u_imm, reg_dest); + ASM_MOV_REG_IMM(emit->as, reg_dest, si->data.u_imm); break; } } @@ -671,7 +583,7 @@ STATIC void emit_fold_stack_top(emit_t *emit, int reg_dest) { si[0] = si[1]; if (si->kind == STACK_VALUE) { // if folded element was on the stack we need to put it in a register - ASM_MOV_LOCAL_TO_REG(emit->as, emit->stack_start + emit->stack_size - 1, reg_dest); + ASM_MOV_REG_LOCAL(emit->as, reg_dest, emit->stack_start + emit->stack_size - 1); si->kind = STACK_REG; si->data.u_reg = reg_dest; } @@ -765,30 +677,30 @@ STATIC void emit_call(emit_t *emit, mp_fun_kind_t fun_kind) { STATIC void emit_call_with_imm_arg(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val, int arg_reg) { need_reg_all(emit); - ASM_MOV_IMM_TO_REG(emit->as, arg_val, arg_reg); + ASM_MOV_REG_IMM(emit->as, arg_reg, arg_val); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } // the first arg is stored in the code aligned on a mp_uint_t boundary STATIC void emit_call_with_imm_arg_aligned(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val, int arg_reg) { need_reg_all(emit); - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, arg_val, arg_reg); + ASM_MOV_REG_ALIGNED_IMM(emit->as, arg_reg, arg_val); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } STATIC void emit_call_with_2_imm_args(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val1, int arg_reg1, mp_int_t arg_val2, int arg_reg2) { need_reg_all(emit); - ASM_MOV_IMM_TO_REG(emit->as, arg_val1, arg_reg1); - ASM_MOV_IMM_TO_REG(emit->as, arg_val2, arg_reg2); + ASM_MOV_REG_IMM(emit->as, arg_reg1, arg_val1); + ASM_MOV_REG_IMM(emit->as, arg_reg2, arg_val2); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } // the first arg is stored in the code aligned on a mp_uint_t boundary STATIC void emit_call_with_3_imm_args_and_first_aligned(emit_t *emit, mp_fun_kind_t fun_kind, mp_int_t arg_val1, int arg_reg1, mp_int_t arg_val2, int arg_reg2, mp_int_t arg_val3, int arg_reg3) { need_reg_all(emit); - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, arg_val1, arg_reg1); - ASM_MOV_IMM_TO_REG(emit->as, arg_val2, arg_reg2); - ASM_MOV_IMM_TO_REG(emit->as, arg_val3, arg_reg3); + ASM_MOV_REG_ALIGNED_IMM(emit->as, arg_reg1, arg_val1); + ASM_MOV_REG_IMM(emit->as, arg_reg2, arg_val2); + ASM_MOV_REG_IMM(emit->as, arg_reg3, arg_val3); ASM_CALL_IND(emit->as, mp_fun_table[fun_kind], fun_kind); } @@ -808,19 +720,19 @@ STATIC void emit_get_stack_pointer_to_reg_for_pop(emit_t *emit, mp_uint_t reg_de si->kind = STACK_VALUE; switch (si->vtype) { case VTYPE_PYOBJ: - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, si->data.u_imm, emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, si->data.u_imm, reg_dest); break; case VTYPE_BOOL: if (si->data.u_imm == 0) { - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, (mp_uint_t)mp_const_false, emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, (mp_uint_t)mp_const_false, reg_dest); } else { - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, (mp_uint_t)mp_const_true, emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, (mp_uint_t)mp_const_true, reg_dest); } si->vtype = VTYPE_PYOBJ; break; case VTYPE_INT: case VTYPE_UINT: - ASM_MOV_IMM_TO_LOCAL_USING(emit->as, (uintptr_t)MP_OBJ_NEW_SMALL_INT(si->data.u_imm), emit->stack_start + emit->stack_size - 1 - i, reg_dest); + ASM_MOV_LOCAL_IMM_VIA(emit->as, emit->stack_start + emit->stack_size - 1 - i, (uintptr_t)MP_OBJ_NEW_SMALL_INT(si->data.u_imm), reg_dest); si->vtype = VTYPE_PYOBJ; break; default: @@ -838,9 +750,9 @@ STATIC void emit_get_stack_pointer_to_reg_for_pop(emit_t *emit, mp_uint_t reg_de stack_info_t *si = &emit->stack_info[emit->stack_size - 1 - i]; if (si->vtype != VTYPE_PYOBJ) { mp_uint_t local_num = emit->stack_start + emit->stack_size - 1 - i; - ASM_MOV_LOCAL_TO_REG(emit->as, local_num, REG_ARG_1); + ASM_MOV_REG_LOCAL(emit->as, REG_ARG_1, local_num); emit_call_with_imm_arg(emit, MP_F_CONVERT_NATIVE_TO_OBJ, si->vtype, REG_ARG_2); // arg2 = type - ASM_MOV_REG_TO_LOCAL(emit->as, REG_RET, local_num); + ASM_MOV_LOCAL_REG(emit->as, local_num, REG_RET); si->vtype = VTYPE_PYOBJ; DEBUG_printf(" convert_native_to_obj(local_num=" UINT_FMT ")\n", local_num); } @@ -848,7 +760,7 @@ STATIC void emit_get_stack_pointer_to_reg_for_pop(emit_t *emit, mp_uint_t reg_de // Adujust the stack for a pop of n_pop items, and load the stack pointer into reg_dest. adjust_stack(emit, -n_pop); - ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, emit->stack_start + emit->stack_size, reg_dest); + ASM_MOV_REG_LOCAL_ADDR(emit->as, reg_dest, emit->stack_start + emit->stack_size); } // vtype of all n_push objects is VTYPE_PYOBJ @@ -858,7 +770,7 @@ STATIC void emit_get_stack_pointer_to_reg_for_push(emit_t *emit, mp_uint_t reg_d emit->stack_info[emit->stack_size + i].kind = STACK_VALUE; emit->stack_info[emit->stack_size + i].vtype = VTYPE_PYOBJ; } - ASM_MOV_LOCAL_ADDR_TO_REG(emit->as, emit->stack_start + emit->stack_size, reg_dest); + ASM_MOV_REG_LOCAL_ADDR(emit->as, reg_dest, emit->stack_start + emit->stack_size); adjust_stack(emit, n_push); } @@ -881,7 +793,7 @@ STATIC void emit_native_import_name(emit_t *emit, qstr qst) { stack_info_t *top = peek_stack(emit, 0); if (top->vtype == VTYPE_PTR_NONE) { emit_pre_pop_discard(emit); - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)mp_const_none, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, (mp_uint_t)mp_const_none); } else { vtype_kind_t vtype_fromlist; emit_pre_pop_reg(emit, &vtype_fromlist, REG_ARG_2); @@ -891,7 +803,7 @@ STATIC void emit_native_import_name(emit_t *emit, qstr qst) { // level argument should be an immediate integer top = peek_stack(emit, 0); assert(top->vtype == VTYPE_INT && top->kind == STACK_IMM); - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)MP_OBJ_NEW_SMALL_INT(top->data.u_imm), REG_ARG_3); + ASM_MOV_REG_IMM(emit->as, REG_ARG_3, (mp_uint_t)MP_OBJ_NEW_SMALL_INT(top->data.u_imm)); emit_pre_pop_discard(emit); } else { @@ -981,7 +893,7 @@ STATIC void emit_native_load_const_str(emit_t *emit, qstr qst) { STATIC void emit_native_load_const_obj(emit_t *emit, mp_obj_t obj) { emit_native_pre(emit); need_reg_single(emit, REG_RET, 0); - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, (mp_uint_t)obj, REG_RET); + ASM_MOV_REG_ALIGNED_IMM(emit->as, REG_RET, (mp_uint_t)obj); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -1006,9 +918,9 @@ STATIC void emit_native_load_fast(emit_t *emit, qstr qst, mp_uint_t local_num) { } else { need_reg_single(emit, REG_TEMP0, 0); if (emit->do_viper_types) { - ASM_MOV_LOCAL_TO_REG(emit->as, local_num - REG_LOCAL_NUM, REG_TEMP0); + ASM_MOV_REG_LOCAL(emit->as, REG_TEMP0, local_num - REG_LOCAL_NUM); } else { - ASM_MOV_LOCAL_TO_REG(emit->as, STATE_START + emit->n_state - 1 - local_num, REG_TEMP0); + ASM_MOV_REG_LOCAL(emit->as, REG_TEMP0, STATE_START + emit->n_state - 1 - local_num); } emit_post_push_reg(emit, vtype, REG_TEMP0); } @@ -1134,7 +1046,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add index to base reg_base = reg_index; } @@ -1151,7 +1063,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 1, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 2*index to base reg_base = reg_index; } @@ -1168,7 +1080,7 @@ STATIC void emit_native_load_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 2, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); ASM_ADD_REG_REG(emit->as, reg_index, reg_base); // add 4*index to base reg_base = reg_index; } @@ -1233,9 +1145,9 @@ STATIC void emit_native_store_fast(emit_t *emit, qstr qst, mp_uint_t local_num) } else { emit_pre_pop_reg(emit, &vtype, REG_TEMP0); if (emit->do_viper_types) { - ASM_MOV_REG_TO_LOCAL(emit->as, REG_TEMP0, local_num - REG_LOCAL_NUM); + ASM_MOV_LOCAL_REG(emit->as, local_num - REG_LOCAL_NUM, REG_TEMP0); } else { - ASM_MOV_REG_TO_LOCAL(emit->as, REG_TEMP0, STATE_START + emit->n_state - 1 - local_num); + ASM_MOV_LOCAL_REG(emit->as, STATE_START + emit->n_state - 1 - local_num, REG_TEMP0); } } emit_post(emit); @@ -1354,7 +1266,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value); #if N_ARM asm_arm_strb_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); return; @@ -1375,7 +1287,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 1, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 1); #if N_ARM asm_arm_strh_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); return; @@ -1396,7 +1308,7 @@ STATIC void emit_native_store_subscr(emit_t *emit) { break; } #endif - ASM_MOV_IMM_TO_REG(emit->as, index_value << 2, reg_index); + ASM_MOV_REG_IMM(emit->as, reg_index, index_value << 2); #if N_ARM asm_arm_str_reg_reg_reg(emit->as, reg_value, reg_base, reg_index); return; @@ -1773,7 +1685,7 @@ STATIC void emit_native_get_iter(emit_t *emit, bool use_stack) { emit_call(emit, MP_F_NATIVE_GETITER); } else { // mp_getiter will allocate the iter_buf on the heap - ASM_MOV_IMM_TO_REG(emit->as, 0, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0); emit_call(emit, MP_F_NATIVE_GETITER); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -1784,7 +1696,7 @@ STATIC void emit_native_for_iter(emit_t *emit, mp_uint_t label) { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_1, MP_OBJ_ITER_BUF_NSLOTS); adjust_stack(emit, MP_OBJ_ITER_BUF_NSLOTS); emit_call(emit, MP_F_NATIVE_ITERNEXT); - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)MP_OBJ_STOP_ITERATION, REG_TEMP1); + ASM_MOV_REG_IMM(emit->as, REG_TEMP1, (mp_uint_t)MP_OBJ_STOP_ITERATION); ASM_JUMP_IF_REG_EQ(emit->as, REG_RET, REG_TEMP1, label); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -2128,12 +2040,12 @@ STATIC void emit_native_make_closure(emit_t *emit, scope_t *scope, mp_uint_t n_c emit_native_pre(emit); if (n_pos_defaults == 0 && n_kw_defaults == 0) { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over); - ASM_MOV_IMM_TO_REG(emit->as, n_closed_over, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, n_closed_over); } else { emit_get_stack_pointer_to_reg_for_pop(emit, REG_ARG_3, n_closed_over + 2); - ASM_MOV_IMM_TO_REG(emit->as, 0x100 | n_closed_over, REG_ARG_2); + ASM_MOV_REG_IMM(emit->as, REG_ARG_2, 0x100 | n_closed_over); } - ASM_MOV_ALIGNED_IMM_TO_REG(emit->as, (mp_uint_t)scope->raw_code, REG_ARG_1); + ASM_MOV_REG_ALIGNED_IMM(emit->as, REG_ARG_1, (mp_uint_t)scope->raw_code); ASM_CALL_IND(emit->as, mp_fun_table[MP_F_MAKE_CLOSURE_FROM_RAW_CODE], MP_F_MAKE_CLOSURE_FROM_RAW_CODE); emit_post_push_reg(emit, VTYPE_PYOBJ, REG_RET); } @@ -2212,9 +2124,9 @@ STATIC void emit_native_return_value(emit_t *emit) { if (peek_vtype(emit, 0) == VTYPE_PTR_NONE) { emit_pre_pop_discard(emit); if (emit->return_vtype == VTYPE_PYOBJ) { - ASM_MOV_IMM_TO_REG(emit->as, (mp_uint_t)mp_const_none, REG_RET); + ASM_MOV_REG_IMM(emit->as, REG_RET, (mp_uint_t)mp_const_none); } else { - ASM_MOV_IMM_TO_REG(emit->as, 0, REG_RET); + ASM_MOV_REG_IMM(emit->as, REG_RET, 0); } } else { vtype_kind_t vtype; diff --git a/python/src/py/emitnthumb.c b/python/src/py/emitnthumb.c new file mode 100644 index 000000000..2b68ca3a1 --- /dev/null +++ b/python/src/py/emitnthumb.c @@ -0,0 +1,15 @@ +// thumb specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_THUMB + +// this is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmthumb.h" + +#define N_THUMB (1) +#define EXPORT_FUN(name) emit_native_thumb_##name +#include "py/emitnative.c" + +#endif diff --git a/python/src/py/emitnx64.c b/python/src/py/emitnx64.c new file mode 100644 index 000000000..b9800f636 --- /dev/null +++ b/python/src/py/emitnx64.c @@ -0,0 +1,15 @@ +// x64 specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_X64 + +// This is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmx64.h" + +#define N_X64 (1) +#define EXPORT_FUN(name) emit_native_x64_##name +#include "py/emitnative.c" + +#endif diff --git a/python/src/py/emitnx86.c b/python/src/py/emitnx86.c new file mode 100644 index 000000000..5d2bbb267 --- /dev/null +++ b/python/src/py/emitnx86.c @@ -0,0 +1,68 @@ +// x86 specific stuff + +#include "py/mpconfig.h" +#include "py/runtime0.h" + +#if MICROPY_EMIT_X86 + +// This is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmx86.h" + +// x86 needs a table to know how many args a given function has +STATIC byte mp_f_n_args[MP_F_NUMBER_OF] = { + [MP_F_CONVERT_OBJ_TO_NATIVE] = 2, + [MP_F_CONVERT_NATIVE_TO_OBJ] = 2, + [MP_F_LOAD_NAME] = 1, + [MP_F_LOAD_GLOBAL] = 1, + [MP_F_LOAD_BUILD_CLASS] = 0, + [MP_F_LOAD_ATTR] = 2, + [MP_F_LOAD_METHOD] = 3, + [MP_F_LOAD_SUPER_METHOD] = 2, + [MP_F_STORE_NAME] = 2, + [MP_F_STORE_GLOBAL] = 2, + [MP_F_STORE_ATTR] = 3, + [MP_F_OBJ_SUBSCR] = 3, + [MP_F_OBJ_IS_TRUE] = 1, + [MP_F_UNARY_OP] = 2, + [MP_F_BINARY_OP] = 3, + [MP_F_BUILD_TUPLE] = 2, + [MP_F_BUILD_LIST] = 2, + [MP_F_LIST_APPEND] = 2, + [MP_F_BUILD_MAP] = 1, + [MP_F_STORE_MAP] = 3, + #if MICROPY_PY_BUILTINS_SET + [MP_F_BUILD_SET] = 2, + [MP_F_STORE_SET] = 2, + #endif + [MP_F_MAKE_FUNCTION_FROM_RAW_CODE] = 3, + [MP_F_NATIVE_CALL_FUNCTION_N_KW] = 3, + [MP_F_CALL_METHOD_N_KW] = 3, + [MP_F_CALL_METHOD_N_KW_VAR] = 3, + [MP_F_NATIVE_GETITER] = 2, + [MP_F_NATIVE_ITERNEXT] = 1, + [MP_F_NLR_PUSH] = 1, + [MP_F_NLR_POP] = 0, + [MP_F_NATIVE_RAISE] = 1, + [MP_F_IMPORT_NAME] = 3, + [MP_F_IMPORT_FROM] = 2, + [MP_F_IMPORT_ALL] = 1, + #if MICROPY_PY_BUILTINS_SLICE + [MP_F_NEW_SLICE] = 3, + #endif + [MP_F_UNPACK_SEQUENCE] = 3, + [MP_F_UNPACK_EX] = 3, + [MP_F_DELETE_NAME] = 1, + [MP_F_DELETE_GLOBAL] = 1, + [MP_F_NEW_CELL] = 1, + [MP_F_MAKE_CLOSURE_FROM_RAW_CODE] = 3, + [MP_F_SETUP_CODE_STATE] = 5, + [MP_F_SMALL_INT_FLOOR_DIVIDE] = 2, + [MP_F_SMALL_INT_MODULO] = 2, +}; + +#define N_X86 (1) +#define EXPORT_FUN(name) emit_native_x86_##name +#include "py/emitnative.c" + +#endif diff --git a/python/src/py/emitnxtensa.c b/python/src/py/emitnxtensa.c new file mode 100644 index 000000000..1a423e21e --- /dev/null +++ b/python/src/py/emitnxtensa.c @@ -0,0 +1,15 @@ +// Xtensa specific stuff + +#include "py/mpconfig.h" + +#if MICROPY_EMIT_XTENSA + +// this is defined so that the assembler exports generic assembler API macros +#define GENERIC_ASM_API (1) +#include "py/asmxtensa.h" + +#define N_XTENSA (1) +#define EXPORT_FUN(name) emit_native_xtensa_##name +#include "py/emitnative.c" + +#endif diff --git a/python/src/py/formatfloat.c b/python/src/py/formatfloat.c index 4228f99ff..dc7fc1d1f 100644 --- a/python/src/py/formatfloat.c +++ b/python/src/py/formatfloat.c @@ -258,7 +258,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch } // It can be that f was right on the edge of an entry in pos_pow needs to be reduced - if (f >= FPCONST(10.0)) { + if ((int)f >= 10) { e += 1; f *= FPCONST(0.1); } @@ -330,7 +330,11 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch // Print the digits of the mantissa for (int i = 0; i < num_digits; ++i, --dec) { int32_t d = (int32_t)f; - *s++ = '0' + d; + if (d < 0) { + *s++ = '0'; + } else { + *s++ = '0' + d; + } if (dec == 0 && prec > 0) { *s++ = '.'; } @@ -341,7 +345,7 @@ int mp_format_float(FPTYPE f, char *buf, size_t buf_size, char fmt, int prec, ch // Round // If we print non-exponential format (i.e. 'f'), but a digit we're going // to round by (e) is too far away, then there's nothing to round. - if ((org_fmt != 'f' || e <= 1) && f >= FPCONST(5.0)) { + if ((org_fmt != 'f' || e <= num_digits) && f >= FPCONST(5.0)) { char *rs = s; rs--; while (1) { diff --git a/python/src/py/gc.c b/python/src/py/gc.c index 9752b3532..84c9918fd 100644 --- a/python/src/py/gc.c +++ b/python/src/py/gc.c @@ -44,6 +44,10 @@ // make this 1 to dump the heap each time it changes #define EXTENSIVE_HEAP_PROFILING (0) +// make this 1 to zero out swept memory to more eagerly +// detect untraced object still in use +#define CLEAR_ON_SWEEP (0) + #define WORDS_PER_BLOCK ((MICROPY_BYTES_PER_GC_BLOCK) / BYTES_PER_WORD) #define BYTES_PER_BLOCK (MICROPY_BYTES_PER_GC_BLOCK) @@ -191,29 +195,22 @@ bool gc_is_locked(void) { && ptr < (void*)MP_STATE_MEM(gc_pool_end) /* must be below end of pool */ \ ) -// ptr should be of type void* -#define VERIFY_MARK_AND_PUSH(ptr) \ - do { \ - if (VERIFY_PTR(ptr)) { \ - size_t _block = BLOCK_FROM_PTR(ptr); \ - if (ATB_GET_KIND(_block) == AT_HEAD) { \ - /* an unmarked head, mark it, and push it on gc stack */ \ - DEBUG_printf("gc_mark(%p)\n", ptr); \ - ATB_HEAD_TO_MARK(_block); \ - if (MP_STATE_MEM(gc_sp) < &MP_STATE_MEM(gc_stack)[MICROPY_ALLOC_GC_STACK_SIZE]) { \ - *MP_STATE_MEM(gc_sp)++ = _block; \ - } else { \ - MP_STATE_MEM(gc_stack_overflow) = 1; \ - } \ - } \ - } \ - } while (0) - -STATIC void gc_drain_stack(void) { - while (MP_STATE_MEM(gc_sp) > MP_STATE_MEM(gc_stack)) { - // pop the next block off the stack - size_t block = *--MP_STATE_MEM(gc_sp); +#ifndef TRACE_MARK +#if DEBUG_PRINT +#define TRACE_MARK(block, ptr) DEBUG_printf("gc_mark(%p)\n", ptr) +#else +#define TRACE_MARK(block, ptr) +#endif +#endif +// Take the given block as the topmost block on the stack. Check all it's +// children: mark the unmarked child blocks and put those newly marked +// blocks on the stack. When all children have been checked, pop off the +// topmost block on the stack and repeat with that one. +STATIC void gc_mark_subtree(size_t block) { + // Start with the block passed in the argument. + size_t sp = 0; + for (;;) { // work out number of consecutive blocks in the chain starting with this one size_t n_blocks = 0; do { @@ -224,22 +221,41 @@ STATIC void gc_drain_stack(void) { void **ptrs = (void**)PTR_FROM_BLOCK(block); for (size_t i = n_blocks * BYTES_PER_BLOCK / sizeof(void*); i > 0; i--, ptrs++) { void *ptr = *ptrs; - VERIFY_MARK_AND_PUSH(ptr); + if (VERIFY_PTR(ptr)) { + // Mark and push this pointer + size_t childblock = BLOCK_FROM_PTR(ptr); + if (ATB_GET_KIND(childblock) == AT_HEAD) { + // an unmarked head, mark it, and push it on gc stack + TRACE_MARK(childblock, ptr); + ATB_HEAD_TO_MARK(childblock); + if (sp < MICROPY_ALLOC_GC_STACK_SIZE) { + MP_STATE_MEM(gc_stack)[sp++] = childblock; + } else { + MP_STATE_MEM(gc_stack_overflow) = 1; + } + } + } } + + // Are there any blocks on the stack? + if (sp == 0) { + break; // No, stack is empty, we're done. + } + + // pop the next block off the stack + block = MP_STATE_MEM(gc_stack)[--sp]; } } STATIC void gc_deal_with_stack_overflow(void) { while (MP_STATE_MEM(gc_stack_overflow)) { MP_STATE_MEM(gc_stack_overflow) = 0; - MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack); // scan entire memory looking for blocks which have been marked but not their children for (size_t block = 0; block < MP_STATE_MEM(gc_alloc_table_byte_len) * BLOCKS_PER_ATB; block++) { // trace (again) if mark bit set if (ATB_GET_KIND(block) == AT_MARK) { - *MP_STATE_MEM(gc_sp)++ = block; - gc_drain_stack(); + gc_mark_subtree(block); } } } @@ -277,7 +293,7 @@ STATIC void gc_sweep(void) { } #endif free_tail = 1; - DEBUG_printf("gc_sweep(%x)\n", PTR_FROM_BLOCK(block)); + DEBUG_printf("gc_sweep(%p)\n", PTR_FROM_BLOCK(block)); #if MICROPY_PY_GC_COLLECT_RETVAL MP_STATE_MEM(gc_collected)++; #endif @@ -286,6 +302,9 @@ STATIC void gc_sweep(void) { case AT_TAIL: if (free_tail) { ATB_ANY_TO_FREE(block); + #if CLEAR_ON_SWEEP + memset((void*)PTR_FROM_BLOCK(block), 0, BYTES_PER_BLOCK); + #endif } break; @@ -304,19 +323,32 @@ void gc_collect_start(void) { MP_STATE_MEM(gc_alloc_amount) = 0; #endif MP_STATE_MEM(gc_stack_overflow) = 0; - MP_STATE_MEM(gc_sp) = MP_STATE_MEM(gc_stack); + // Trace root pointers. This relies on the root pointers being organised // correctly in the mp_state_ctx structure. We scan nlr_top, dict_locals, // dict_globals, then the root pointer section of mp_state_vm. void **ptrs = (void**)(void*)&mp_state_ctx; gc_collect_root(ptrs, offsetof(mp_state_ctx_t, vm.qstr_last_chunk) / sizeof(void*)); + + #if MICROPY_ENABLE_PYSTACK + // Trace root pointers from the Python stack. + ptrs = (void**)(void*)MP_STATE_THREAD(pystack_start); + gc_collect_root(ptrs, (MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start)) / sizeof(void*)); + #endif } void gc_collect_root(void **ptrs, size_t len) { for (size_t i = 0; i < len; i++) { void *ptr = ptrs[i]; - VERIFY_MARK_AND_PUSH(ptr); - gc_drain_stack(); + if (VERIFY_PTR(ptr)) { + size_t block = BLOCK_FROM_PTR(ptr); + if (ATB_GET_KIND(block) == AT_HEAD) { + // An unmarked head: mark it, and mark all its children + TRACE_MARK(block, ptr); + ATB_HEAD_TO_MARK(block); + gc_mark_subtree(block); + } + } } } @@ -628,27 +660,18 @@ void *gc_realloc(void *ptr_in, size_t n_bytes, bool allow_move) { void *ptr = ptr_in; - // sanity check the ptr - if (!VERIFY_PTR(ptr)) { - return NULL; - } - - // get first block - size_t block = BLOCK_FROM_PTR(ptr); - GC_ENTER(); - // sanity check the ptr is pointing to the head of a block - if (ATB_GET_KIND(block) != AT_HEAD) { - GC_EXIT(); - return NULL; - } - if (MP_STATE_MEM(gc_lock_depth) > 0) { GC_EXIT(); return NULL; } + // get the GC block number corresponding to this pointer + assert(VERIFY_PTR(ptr)); + size_t block = BLOCK_FROM_PTR(ptr); + assert(ATB_GET_KIND(block) == AT_HEAD); + // compute number of new blocks that are requested size_t new_blocks = (n_bytes + BYTES_PER_BLOCK - 1) / BYTES_PER_BLOCK; diff --git a/python/src/py/makeqstrdata.py b/python/src/py/makeqstrdata.py index 38fde1a9c..3c0a60909 100644 --- a/python/src/py/makeqstrdata.py +++ b/python/src/py/makeqstrdata.py @@ -114,6 +114,9 @@ def parse_input_headers(infiles): if ident == "": # Sort empty qstr above all still order = -200000 + elif ident == "__dir__": + # Put __dir__ after empty qstr for builtin dir() to work + order = -190000 elif ident.startswith("__"): order -= 100000 qstrs[ident] = (order, ident, qstr) diff --git a/python/src/py/malloc.c b/python/src/py/malloc.c index ea1d4c4b9..ba5c952f3 100644 --- a/python/src/py/malloc.c +++ b/python/src/py/malloc.c @@ -39,6 +39,9 @@ #endif #if MICROPY_MEM_STATS +#if !MICROPY_MALLOC_USES_ALLOCATED_SIZE +#error MICROPY_MEM_STATS requires MICROPY_MALLOC_USES_ALLOCATED_SIZE +#endif #define UPDATE_PEAK() { if (MP_STATE_MEM(current_bytes_allocated) > MP_STATE_MEM(peak_bytes_allocated)) MP_STATE_MEM(peak_bytes_allocated) = MP_STATE_MEM(current_bytes_allocated); } #endif @@ -114,9 +117,6 @@ void *m_malloc_with_finaliser(size_t num_bytes) { void *m_malloc0(size_t num_bytes) { void *ptr = m_malloc(num_bytes); - if (ptr == NULL && num_bytes != 0) { - m_malloc_fail(num_bytes); - } // If this config is set then the GC clears all memory, so we don't need to. #if !MICROPY_GC_CONSERVATIVE_CLEAR memset(ptr, 0, num_bytes); @@ -144,7 +144,11 @@ void *m_realloc(void *ptr, size_t new_num_bytes) { MP_STATE_MEM(current_bytes_allocated) += diff; UPDATE_PEAK(); #endif + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, old_num_bytes, new_num_bytes, new_ptr); + #else + DEBUG_printf("realloc %p, %d : %p\n", ptr, new_num_bytes, new_ptr); + #endif return new_ptr; } @@ -168,7 +172,11 @@ void *m_realloc_maybe(void *ptr, size_t new_num_bytes, bool allow_move) { UPDATE_PEAK(); } #endif + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, old_num_bytes, new_num_bytes, new_ptr); + #else + DEBUG_printf("realloc %p, %d, %d : %p\n", ptr, new_num_bytes, new_ptr); + #endif return new_ptr; } @@ -181,7 +189,11 @@ void m_free(void *ptr) { #if MICROPY_MEM_STATS MP_STATE_MEM(current_bytes_allocated) -= num_bytes; #endif + #if MICROPY_MALLOC_USES_ALLOCATED_SIZE DEBUG_printf("free %p, %d\n", ptr, num_bytes); + #else + DEBUG_printf("free %p\n", ptr); + #endif } #if MICROPY_MEM_STATS diff --git a/python/src/py/map.c b/python/src/py/map.c index 4f76b9b16..6abf4853f 100644 --- a/python/src/py/map.c +++ b/python/src/py/map.c @@ -33,6 +33,13 @@ #include "py/misc.h" #include "py/runtime.h" +#if MICROPY_DEBUG_VERBOSE // print debugging info +#define DEBUG_PRINT (1) +#else // don't print debugging info +#define DEBUG_PRINT (0) +#define DEBUG_printf(...) (void)0 +#endif + // Fixed empty map. Useful when need to call kw-receiving functions // without any keywords from C, etc. const mp_map_t mp_const_empty_map = { @@ -114,6 +121,7 @@ void mp_map_clear(mp_map_t *map) { STATIC void mp_map_rehash(mp_map_t *map) { size_t old_alloc = map->alloc; size_t new_alloc = get_hash_alloc_greater_or_equal_to(map->alloc + 1); + DEBUG_printf("mp_map_rehash(%p): " UINT_FMT " -> " UINT_FMT "\n", map, old_alloc, new_alloc); mp_map_elem_t *old_table = map->table; mp_map_elem_t *new_table = m_new0(mp_map_elem_t, new_alloc); // If we reach this point, table resizing succeeded, now we can edit the old map. @@ -162,6 +170,7 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t if (map->is_ordered) { for (mp_map_elem_t *elem = &map->table[0], *top = &map->table[map->used]; elem < top; elem++) { if (elem->key == index || (!compare_only_ptrs && mp_obj_equal(elem->key, index))) { + #if MICROPY_PY_COLLECTIONS_ORDEREDDICT if (MP_UNLIKELY(lookup_kind == MP_MAP_LOOKUP_REMOVE_IF_FOUND)) { // remove the found element by moving the rest of the array down mp_obj_t value = elem->value; @@ -172,9 +181,11 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t elem->key = MP_OBJ_NULL; elem->value = value; } + #endif return elem; } } + #if MICROPY_PY_COLLECTIONS_ORDEREDDICT if (MP_LIKELY(lookup_kind != MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)) { return NULL; } @@ -190,6 +201,9 @@ mp_map_elem_t *mp_map_lookup(mp_map_t *map, mp_obj_t index, mp_map_lookup_kind_t map->all_keys_are_qstrs = 0; } return elem; + #else + return NULL; + #endif } // map is a hash table (not an ordered array), so do a hash lookup diff --git a/python/src/py/misc.h b/python/src/py/misc.h index b9f2dae90..72560da1e 100644 --- a/python/src/py/misc.h +++ b/python/src/py/misc.h @@ -63,8 +63,10 @@ typedef unsigned int uint; #define m_new_obj_var_maybe(obj_type, var_type, var_num) ((obj_type*)m_malloc_maybe(sizeof(obj_type) + sizeof(var_type) * (var_num))) #if MICROPY_ENABLE_FINALISER #define m_new_obj_with_finaliser(type) ((type*)(m_malloc_with_finaliser(sizeof(type)))) +#define m_new_obj_var_with_finaliser(type, var_type, var_num) ((type*)m_malloc_with_finaliser(sizeof(type) + sizeof(var_type) * (var_num))) #else #define m_new_obj_with_finaliser(type) m_new_obj(type) +#define m_new_obj_var_with_finaliser(type, var_type, var_num) m_new_obj_var(type, var_type, var_num) #endif #if MICROPY_MALLOC_USES_ALLOCATED_SIZE #define m_renew(type, ptr, old_num, new_num) ((type*)(m_realloc((ptr), sizeof(type) * (old_num), sizeof(type) * (new_num)))) @@ -119,8 +121,15 @@ typedef uint32_t unichar; typedef uint unichar; #endif +#if MICROPY_PY_BUILTINS_STR_UNICODE unichar utf8_get_char(const byte *s); const byte *utf8_next_char(const byte *s); +size_t utf8_charlen(const byte *str, size_t len); +#else +static inline unichar utf8_get_char(const byte *s) { return *s; } +static inline const byte *utf8_next_char(const byte *s) { return s + 1; } +static inline size_t utf8_charlen(const byte *str, size_t len) { (void)str; return len; } +#endif bool unichar_isspace(unichar c); bool unichar_isalpha(unichar c); @@ -133,7 +142,6 @@ bool unichar_islower(unichar c); unichar unichar_tolower(unichar c); unichar unichar_toupper(unichar c); mp_uint_t unichar_xdigit_value(unichar c); -mp_uint_t unichar_charlen(const char *str, mp_uint_t len); #define UTF8_IS_NONASCII(ch) ((ch) & 0x80) #define UTF8_IS_CONT(ch) (((ch) & 0xC0) == 0x80) @@ -196,20 +204,6 @@ int DEBUG_printf(const char *fmt, ...); extern mp_uint_t mp_verbose_flag; -// This is useful for unicode handling. Some CPU archs has -// special instructions for efficient implementation of this -// function (e.g. CLZ on ARM). -// NOTE: this function is unused at the moment -#ifndef count_lead_ones -static inline mp_uint_t count_lead_ones(byte val) { - mp_uint_t c = 0; - for (byte mask = 0x80; val & mask; mask >>= 1) { - c++; - } - return c; -} -#endif - /** float internals *************/ #if MICROPY_PY_BUILTINS_FLOAT diff --git a/python/src/py/modbuiltins.c b/python/src/py/modbuiltins.c index 65c03d523..0d511338b 100644 --- a/python/src/py/modbuiltins.c +++ b/python/src/py/modbuiltins.c @@ -137,7 +137,7 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_callable_obj, mp_builtin_callable); STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { #if MICROPY_PY_BUILTINS_STR_UNICODE mp_uint_t c = mp_obj_get_int(o_in); - char str[4]; + uint8_t str[4]; int len = 0; if (c < 0x80) { *str = c; len = 1; @@ -159,12 +159,12 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { } else { mp_raise_ValueError("chr() arg not in range(0x110000)"); } - return mp_obj_new_str(str, len, true); + return mp_obj_new_str_via_qstr((char*)str, len); #else mp_int_t ord = mp_obj_get_int(o_in); if (0 <= ord && ord <= 0xff) { - char str[1] = {ord}; - return mp_obj_new_str(str, 1, true); + uint8_t str[1] = {ord}; + return mp_obj_new_str_via_qstr((char*)str, 1); } else { mp_raise_ValueError("chr() arg not in range(256)"); } @@ -173,46 +173,31 @@ STATIC mp_obj_t mp_builtin_chr(mp_obj_t o_in) { MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_chr_obj, mp_builtin_chr); STATIC mp_obj_t mp_builtin_dir(size_t n_args, const mp_obj_t *args) { - // TODO make this function more general and less of a hack - - mp_obj_dict_t *dict = NULL; - mp_map_t *members = NULL; - if (n_args == 0) { - // make a list of names in the local name space - dict = mp_locals_get(); - } else { // n_args == 1 - // make a list of names in the given object - if (MP_OBJ_IS_TYPE(args[0], &mp_type_module)) { - dict = mp_obj_module_get_globals(args[0]); - } else { - mp_obj_type_t *type; - if (MP_OBJ_IS_TYPE(args[0], &mp_type_type)) { - type = MP_OBJ_TO_PTR(args[0]); - } else { - type = mp_obj_get_type(args[0]); - } - if (type->locals_dict != NULL && type->locals_dict->base.type == &mp_type_dict) { - dict = type->locals_dict; - } - } - if (mp_obj_is_instance_type(mp_obj_get_type(args[0]))) { - mp_obj_instance_t *inst = MP_OBJ_TO_PTR(args[0]); - members = &inst->members; - } - } - mp_obj_t dir = mp_obj_new_list(0, NULL); - if (dict != NULL) { + if (n_args == 0) { + // Make a list of names in the local namespace + mp_obj_dict_t *dict = mp_locals_get(); for (size_t i = 0; i < dict->map.alloc; i++) { if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { mp_obj_list_append(dir, dict->map.table[i].key); } } - } - if (members != NULL) { - for (size_t i = 0; i < members->alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(members, i)) { - mp_obj_list_append(dir, members->table[i].key); + } else { // n_args == 1 + // Make a list of names in the given object + // Implemented by probing all possible qstrs with mp_load_method_maybe + size_t nqstr = QSTR_TOTAL(); + for (size_t i = MP_QSTR_ + 1; i < nqstr; ++i) { + mp_obj_t dest[2]; + mp_load_method_protected(args[0], i, dest, false); + if (dest[0] != MP_OBJ_NULL) { + #if MICROPY_PY_ALL_SPECIAL_METHODS + // Support for __dir__: see if we can dispatch to this special method + // This relies on MP_QSTR__dir__ being first after MP_QSTR_ + if (i == MP_QSTR___dir__ && dest[1] != MP_OBJ_NULL) { + return mp_call_method_n_kw(0, 0, dest); + } + #endif + mp_obj_list_append(dir, MP_OBJ_NEW_QSTR(i)); } } } @@ -343,19 +328,19 @@ MP_DEFINE_CONST_FUN_OBJ_1(mp_builtin_oct_obj, mp_builtin_oct); STATIC mp_obj_t mp_builtin_ord(mp_obj_t o_in) { size_t len; - const char *str = mp_obj_str_get_data(o_in, &len); + const byte *str = (const byte*)mp_obj_str_get_data(o_in, &len); #if MICROPY_PY_BUILTINS_STR_UNICODE if (MP_OBJ_IS_STR(o_in)) { - len = unichar_charlen(str, len); + len = utf8_charlen(str, len); if (len == 1) { - return mp_obj_new_int(utf8_get_char((const byte*)str)); + return mp_obj_new_int(utf8_get_char(str)); } } else #endif { // a bytes object, or a str without unicode support (don't sign extend the char) if (len == 1) { - return MP_OBJ_NEW_SMALL_INT(((const byte*)str)[0]); + return MP_OBJ_NEW_SMALL_INT(str[0]); } } @@ -383,46 +368,52 @@ STATIC mp_obj_t mp_builtin_pow(size_t n_args, const mp_obj_t *args) { } MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_builtin_pow_obj, 2, 3, mp_builtin_pow); -STATIC mp_obj_t mp_builtin_print(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { - mp_map_elem_t *sep_elem = mp_map_lookup(kwargs, MP_OBJ_NEW_QSTR(MP_QSTR_sep), MP_MAP_LOOKUP); - mp_map_elem_t *end_elem = mp_map_lookup(kwargs, MP_OBJ_NEW_QSTR(MP_QSTR_end), MP_MAP_LOOKUP); - const char *sep_data = " "; - size_t sep_len = 1; - const char *end_data = "\n"; - size_t end_len = 1; - if (sep_elem != NULL && sep_elem->value != mp_const_none) { - sep_data = mp_obj_str_get_data(sep_elem->value, &sep_len); - } - if (end_elem != NULL && end_elem->value != mp_const_none) { - end_data = mp_obj_str_get_data(end_elem->value, &end_len); - } - #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - void *stream_obj = &mp_sys_stdout_obj; - mp_map_elem_t *file_elem = mp_map_lookup(kwargs, MP_OBJ_NEW_QSTR(MP_QSTR_file), MP_MAP_LOOKUP); - if (file_elem != NULL && file_elem->value != mp_const_none) { - stream_obj = MP_OBJ_TO_PTR(file_elem->value); // XXX may not be a concrete object - } +STATIC mp_obj_t mp_builtin_print(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + enum { ARG_sep, ARG_end, ARG_file }; + static const mp_arg_t allowed_args[] = { + { MP_QSTR_sep, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR__space_)} }, + { MP_QSTR_end, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_QSTR(MP_QSTR__0x0a_)} }, + #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES + { MP_QSTR_file, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_rom_obj = MP_ROM_PTR(&mp_sys_stdout_obj)} }, + #endif + }; - mp_print_t print = {stream_obj, mp_stream_write_adaptor}; + // parse args (a union is used to reduce the amount of C stack that is needed) + union { + mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)]; + size_t len[2]; + } u; + mp_arg_parse_all(0, NULL, kw_args, MP_ARRAY_SIZE(allowed_args), allowed_args, u.args); + + #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES + // TODO file may not be a concrete object (eg it could be a small-int) + mp_print_t print = {MP_OBJ_TO_PTR(u.args[ARG_file].u_obj), mp_stream_write_adaptor}; #endif + + // extract the objects first because we are going to use the other part of the union + mp_obj_t sep = u.args[ARG_sep].u_obj; + mp_obj_t end = u.args[ARG_end].u_obj; + const char *sep_data = mp_obj_str_get_data(sep, &u.len[0]); + const char *end_data = mp_obj_str_get_data(end, &u.len[1]); + for (size_t i = 0; i < n_args; i++) { if (i > 0) { #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - mp_stream_write_adaptor(stream_obj, sep_data, sep_len); + mp_stream_write_adaptor(print.data, sep_data, u.len[0]); #else - mp_print_strn(&mp_plat_print, sep_data, sep_len, 0, 0, 0); + mp_print_strn(&mp_plat_print, sep_data, u.len[0], 0, 0, 0); #endif } #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - mp_obj_print_helper(&print, args[i], PRINT_STR); + mp_obj_print_helper(&print, pos_args[i], PRINT_STR); #else - mp_obj_print_helper(&mp_plat_print, args[i], PRINT_STR); + mp_obj_print_helper(&mp_plat_print, pos_args[i], PRINT_STR); #endif } #if MICROPY_PY_IO && MICROPY_PY_SYS_STDFILES - mp_stream_write_adaptor(stream_obj, end_data, end_len); + mp_stream_write_adaptor(print.data, end_data, u.len[1]); #else - mp_print_strn(&mp_plat_print, end_data, end_len, 0, 0, 0); + mp_print_strn(&mp_plat_print, end_data, u.len[1], 0, 0, 0); #endif return mp_const_none; } @@ -457,16 +448,14 @@ STATIC mp_obj_t mp_builtin_round(size_t n_args, const mp_obj_t *args) { return o_in; } #if MICROPY_PY_BUILTINS_FLOAT - mp_int_t num_dig = 0; + mp_float_t val = mp_obj_get_float(o_in); if (n_args > 1) { - num_dig = mp_obj_get_int(args[1]); - mp_float_t val = mp_obj_get_float(o_in); + mp_int_t num_dig = mp_obj_get_int(args[1]); mp_float_t mult = MICROPY_FLOAT_C_FUN(pow)(10, num_dig); // TODO may lead to overflow mp_float_t rounded = MICROPY_FLOAT_C_FUN(nearbyint)(val * mult) / mult; return mp_obj_new_float(rounded); } - mp_float_t val = mp_obj_get_float(o_in); mp_float_t rounded = MICROPY_FLOAT_C_FUN(nearbyint)(val); return mp_obj_new_int_from_float(rounded); #else @@ -543,14 +532,8 @@ MP_DEFINE_CONST_FUN_OBJ_2(mp_builtin_delattr_obj, mp_builtin_delattr); STATIC mp_obj_t mp_builtin_hasattr(mp_obj_t object_in, mp_obj_t attr_in) { qstr attr = mp_obj_str_get_qstr(attr_in); - mp_obj_t dest[2]; - // TODO: https://docs.python.org/3/library/functions.html?highlight=hasattr#hasattr - // explicitly says "This is implemented by calling getattr(object, name) and seeing - // whether it raises an AttributeError or not.", so we should explicitly wrap this - // in nlr_push and handle exception. - mp_load_method_maybe(object_in, attr, dest); - + mp_load_method_protected(object_in, attr, dest, false); return mp_obj_new_bool(dest[0] != MP_OBJ_NULL); } MP_DEFINE_CONST_FUN_OBJ_2(mp_builtin_hasattr_obj, mp_builtin_hasattr); diff --git a/python/src/py/modcollections.c b/python/src/py/modcollections.c index 1a1560387..bb6488471 100644 --- a/python/src/py/modcollections.c +++ b/python/src/py/modcollections.c @@ -30,6 +30,9 @@ STATIC const mp_rom_map_elem_t mp_module_collections_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_ucollections) }, + #if MICROPY_PY_COLLECTIONS_DEQUE + { MP_ROM_QSTR(MP_QSTR_deque), MP_ROM_PTR(&mp_type_deque) }, + #endif { MP_ROM_QSTR(MP_QSTR_namedtuple), MP_ROM_PTR(&mp_namedtuple_obj) }, #if MICROPY_PY_COLLECTIONS_ORDEREDDICT { MP_ROM_QSTR(MP_QSTR_OrderedDict), MP_ROM_PTR(&mp_type_ordereddict) }, diff --git a/python/src/py/modio.c b/python/src/py/modio.c index 353a00286..3a5c69c4c 100644 --- a/python/src/py/modio.c +++ b/python/src/py/modio.c @@ -131,7 +131,7 @@ STATIC const mp_obj_type_t bufwriter_type = { }; #endif // MICROPY_PY_IO_BUFFEREDWRITER -#if MICROPY_MODULE_FROZEN_STR +#if MICROPY_PY_IO_RESOURCE_STREAM STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { VSTR_FIXED(path_buf, MICROPY_ALLOC_PATH_MAX); size_t len; @@ -176,10 +176,10 @@ STATIC mp_obj_t resource_stream(mp_obj_t package_in, mp_obj_t path_in) { return MP_OBJ_FROM_PTR(o); } - mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len, false); + mp_obj_t path_out = mp_obj_new_str(path_buf.buf, path_buf.len); return mp_builtin_open(1, &path_out, (mp_map_t*)&mp_const_empty_map); } -MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); +STATIC MP_DEFINE_CONST_FUN_OBJ_2(resource_stream_obj, resource_stream); #endif STATIC const mp_rom_map_elem_t mp_module_io_globals_table[] = { diff --git a/python/src/py/modmicropython.c b/python/src/py/modmicropython.c index 2aac53adc..864d1a5c5 100644 --- a/python/src/py/modmicropython.c +++ b/python/src/py/modmicropython.c @@ -35,6 +35,7 @@ // Various builtins specific to MicroPython runtime, // living in micropython module +#if MICROPY_ENABLE_COMPILER STATIC mp_obj_t mp_micropython_opt_level(size_t n_args, const mp_obj_t *args) { if (n_args == 0) { return MP_OBJ_NEW_SMALL_INT(MP_STATE_VM(mp_optimise_value)); @@ -44,6 +45,7 @@ STATIC mp_obj_t mp_micropython_opt_level(size_t n_args, const mp_obj_t *args) { } } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_opt_level_obj, 0, 1, mp_micropython_opt_level); +#endif #if MICROPY_PY_MICROPYTHON_MEM_INFO @@ -103,14 +105,21 @@ STATIC mp_obj_t mp_micropython_qstr_info(size_t n_args, const mp_obj_t *args) { } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(mp_micropython_qstr_info_obj, 0, 1, mp_micropython_qstr_info); -#if MICROPY_STACK_CHECK +#endif // MICROPY_PY_MICROPYTHON_MEM_INFO + +#if MICROPY_PY_MICROPYTHON_STACK_USE STATIC mp_obj_t mp_micropython_stack_use(void) { return MP_OBJ_NEW_SMALL_INT(mp_stack_usage()); } STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_stack_use_obj, mp_micropython_stack_use); #endif -#endif // MICROPY_PY_MICROPYTHON_MEM_INFO +#if MICROPY_ENABLE_PYSTACK +STATIC mp_obj_t mp_micropython_pystack_use(void) { + return MP_OBJ_NEW_SMALL_INT(mp_pystack_usage()); +} +STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_micropython_pystack_use_obj, mp_micropython_pystack_use); +#endif #if MICROPY_ENABLE_GC STATIC mp_obj_t mp_micropython_heap_lock(void) { @@ -151,7 +160,9 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_2(mp_micropython_schedule_obj, mp_micropython_sch STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_micropython) }, { MP_ROM_QSTR(MP_QSTR_const), MP_ROM_PTR(&mp_identity_obj) }, + #if MICROPY_ENABLE_COMPILER { MP_ROM_QSTR(MP_QSTR_opt_level), MP_ROM_PTR(&mp_micropython_opt_level_obj) }, + #endif #if MICROPY_PY_MICROPYTHON_MEM_INFO #if MICROPY_MEM_STATS { MP_ROM_QSTR(MP_QSTR_mem_total), MP_ROM_PTR(&mp_micropython_mem_total_obj) }, @@ -160,13 +171,16 @@ STATIC const mp_rom_map_elem_t mp_module_micropython_globals_table[] = { #endif { MP_ROM_QSTR(MP_QSTR_mem_info), MP_ROM_PTR(&mp_micropython_mem_info_obj) }, { MP_ROM_QSTR(MP_QSTR_qstr_info), MP_ROM_PTR(&mp_micropython_qstr_info_obj) }, - #if MICROPY_STACK_CHECK +#endif + #if MICROPY_PY_MICROPYTHON_STACK_USE { MP_ROM_QSTR(MP_QSTR_stack_use), MP_ROM_PTR(&mp_micropython_stack_use_obj) }, #endif -#endif #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && (MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0) { MP_ROM_QSTR(MP_QSTR_alloc_emergency_exception_buf), MP_ROM_PTR(&mp_alloc_emergency_exception_buf_obj) }, #endif + #if MICROPY_ENABLE_PYSTACK + { MP_ROM_QSTR(MP_QSTR_pystack_use), MP_ROM_PTR(&mp_micropython_pystack_use_obj) }, + #endif #if MICROPY_ENABLE_GC { MP_ROM_QSTR(MP_QSTR_heap_lock), MP_ROM_PTR(&mp_micropython_heap_lock_obj) }, { MP_ROM_QSTR(MP_QSTR_heap_unlock), MP_ROM_PTR(&mp_micropython_heap_unlock_obj) }, diff --git a/python/src/py/modsys.c b/python/src/py/modsys.c index 84a4eb0f4..609f8b454 100644 --- a/python/src/py/modsys.c +++ b/python/src/py/modsys.c @@ -140,10 +140,12 @@ STATIC mp_obj_t mp_sys_exc_info(void) { MP_DEFINE_CONST_FUN_OBJ_0(mp_sys_exc_info_obj, mp_sys_exc_info); #endif +#if MICROPY_PY_SYS_GETSIZEOF STATIC mp_obj_t mp_sys_getsizeof(mp_obj_t obj) { return mp_unary_op(MP_UNARY_OP_SIZEOF, obj); } -MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_getsizeof_obj, mp_sys_getsizeof); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_sys_getsizeof_obj, mp_sys_getsizeof); +#endif STATIC const mp_rom_map_elem_t mp_module_sys_globals_table[] = { { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_sys) }, diff --git a/python/src/py/modthread.c b/python/src/py/modthread.c index cb071d0f8..61ada5035 100644 --- a/python/src/py/modthread.c +++ b/python/src/py/modthread.c @@ -165,6 +165,12 @@ STATIC void *thread_entry(void *args_in) { mp_stack_set_top(&ts + 1); // need to include ts in root-pointer scan mp_stack_set_limit(args->stack_size); + #if MICROPY_ENABLE_PYSTACK + // TODO threading and pystack is not fully supported, for now just make a small stack + mp_obj_t mini_pystack[128]; + mp_pystack_init(mini_pystack, &mini_pystack[128]); + #endif + // set locals and globals from the calling context mp_locals_set(args->dict_locals); mp_globals_set(args->dict_globals); diff --git a/python/src/py/mpconfig.h b/python/src/py/mpconfig.h index 6a32ea2a6..532b54ab0 100644 --- a/python/src/py/mpconfig.h +++ b/python/src/py/mpconfig.h @@ -79,7 +79,7 @@ // - seeeeeee eeeeffff ffffffff ffffffff ffffffff ffffffff ffffffff ffffffff 64-bit fp, e != 0x7ff // - s1111111 11110000 00000000 00000000 00000000 00000000 00000000 00000000 +/- inf // - 01111111 11111000 00000000 00000000 00000000 00000000 00000000 00000000 normalised nan -// - 01111111 11111101 00000000 00000000 iiiiiiii iiiiiiii iiiiiiii iiiiiii1 small int +// - 01111111 11111101 iiiiiiii iiiiiiii iiiiiiii iiiiiiii iiiiiiii iiiiiii1 small int // - 01111111 11111110 00000000 00000000 qqqqqqqq qqqqqqqq qqqqqqqq qqqqqqq1 str // - 01111111 11111100 00000000 00000000 pppppppp pppppppp pppppppp pppppp00 ptr (4 byte alignment) // Stored as O = R + 0x8004000000000000, retrieved as R = O - 0x8004000000000000. @@ -347,7 +347,7 @@ #endif // Whether to enable optimisation of: a, b, c = d, e, f -// Cost 156 bytes (Thumb2) +// Requires MICROPY_COMP_DOUBLE_TUPLE_ASSIGN and costs 68 bytes (Thumb2) #ifndef MICROPY_COMP_TRIPLE_TUPLE_ASSIGN #define MICROPY_COMP_TRIPLE_TUPLE_ASSIGN (0) #endif @@ -405,6 +405,13 @@ /*****************************************************************************/ /* Python internal features */ +// Whether to enable import of external modules +// When disabled, only importing of built-in modules is supported +// When enabled, a port must implement mp_import_stat (among other things) +#ifndef MICROPY_ENABLE_EXTERNAL_IMPORT +#define MICROPY_ENABLE_EXTERNAL_IMPORT (1) +#endif + // Whether to use the POSIX reader for importing files #ifndef MICROPY_READER_POSIX #define MICROPY_READER_POSIX (0) @@ -441,6 +448,17 @@ #define MICROPY_ENABLE_FINALISER (0) #endif +// Whether to enable a separate allocator for the Python stack. +// If enabled then the code must call mp_pystack_init before mp_init. +#ifndef MICROPY_ENABLE_PYSTACK +#define MICROPY_ENABLE_PYSTACK (0) +#endif + +// Number of bytes that memory returned by mp_pystack_alloc will be aligned by. +#ifndef MICROPY_PYSTACK_ALIGN +#define MICROPY_PYSTACK_ALIGN (8) +#endif + // Whether to check C stack usage. C stack used for calling Python functions, // etc. Not checking means segfault on overflow. #ifndef MICROPY_STACK_CHECK @@ -666,6 +684,13 @@ typedef double mp_float_t; /*****************************************************************************/ /* Fine control over Python builtins, classes, modules, etc */ +// Whether to support multiple inheritance of Python classes. Multiple +// inheritance makes some C functions inherently recursive, and adds a bit of +// code overhead. +#ifndef MICROPY_MULTIPLE_INHERITANCE +#define MICROPY_MULTIPLE_INHERITANCE (1) +#endif + // Whether to implement attributes on functions #ifndef MICROPY_PY_FUNCTION_ATTRS #define MICROPY_PY_FUNCTION_ATTRS (0) @@ -688,6 +713,15 @@ typedef double mp_float_t; #define MICROPY_PY_ASYNC_AWAIT (1) #endif +// Non-standard .pend_throw() method for generators, allowing for +// Future-like behavior with respect to exception handling: an +// exception set with .pend_throw() will activate on the next call +// to generator's .send() or .__next__(). (This is useful to implement +// async schedulers.) +#ifndef MICROPY_PY_GENERATOR_PEND_THROW +#define MICROPY_PY_GENERATOR_PEND_THROW (1) +#endif + // Issue a warning when comparing str and bytes objects #ifndef MICROPY_PY_STR_BYTES_CMP_WARN #define MICROPY_PY_STR_BYTES_CMP_WARN (0) @@ -760,6 +794,14 @@ typedef double mp_float_t; #define MICROPY_PY_BUILTINS_RANGE_ATTRS (1) #endif +// Whether to support binary ops [only (in)equality is defined] between range +// objects. With this option disabled all range objects that are not exactly +// the same object will compare as not-equal. With it enabled the semantics +// match CPython and ranges are equal if they yield the same sequence of items. +#ifndef MICROPY_PY_BUILTINS_RANGE_BINOP +#define MICROPY_PY_BUILTINS_RANGE_BINOP (0) +#endif + // Whether to support timeout exceptions (like socket.timeout) #ifndef MICROPY_PY_BUILTINS_TIMEOUTERROR #define MICROPY_PY_BUILTINS_TIMEOUTERROR (0) @@ -865,6 +907,11 @@ typedef double mp_float_t; #define MICROPY_PY_MICROPYTHON_MEM_INFO (0) #endif +// Whether to provide "micropython.stack_use" function +#ifndef MICROPY_PY_MICROPYTHON_STACK_USE +#define MICROPY_PY_MICROPYTHON_STACK_USE (MICROPY_PY_MICROPYTHON_MEM_INFO) +#endif + // Whether to provide "array" module. Note that large chunk of the // underlying code is shared with "bytearray" builtin type, so to // get real savings, it should be disabled too. @@ -889,11 +936,21 @@ typedef double mp_float_t; #define MICROPY_PY_COLLECTIONS (1) #endif +// Whether to provide "ucollections.deque" type +#ifndef MICROPY_PY_COLLECTIONS_DEQUE +#define MICROPY_PY_COLLECTIONS_DEQUE (0) +#endif + // Whether to provide "collections.OrderedDict" type #ifndef MICROPY_PY_COLLECTIONS_ORDEREDDICT #define MICROPY_PY_COLLECTIONS_ORDEREDDICT (0) #endif +// Whether to provide the _asdict function for namedtuple +#ifndef MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT +#define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (0) +#endif + // Whether to provide "math" module #ifndef MICROPY_PY_MATH #define MICROPY_PY_MATH (1) @@ -926,7 +983,11 @@ typedef double mp_float_t; // Whether to provide "uio.resource_stream()" function with // the semantics of CPython's pkg_resources.resource_stream() -// (allows to access resources in frozen packages). +// (allows to access binary resources in frozen source packages). +// Note that the same functionality can be achieved in "pure +// Python" by prepocessing binary resources into Python source +// and bytecode-freezing it (with a simple helper module available +// e.g. in micropython-lib). #ifndef MICROPY_PY_IO_RESOURCE_STREAM #define MICROPY_PY_IO_RESOURCE_STREAM (0) #endif @@ -1293,4 +1354,24 @@ typedef double mp_float_t; #define MP_UNLIKELY(x) __builtin_expect((x), 0) #endif +#ifndef MP_HTOBE16 +#if MP_ENDIANNESS_LITTLE +# define MP_HTOBE16(x) ((uint16_t)( (((x) & 0xff) << 8) | (((x) >> 8) & 0xff) )) +# define MP_BE16TOH(x) MP_HTOBE16(x) +#else +# define MP_HTOBE16(x) (x) +# define MP_BE16TOH(x) (x) +#endif +#endif + +#ifndef MP_HTOBE32 +#if MP_ENDIANNESS_LITTLE +# define MP_HTOBE32(x) ((uint32_t)( (((x) & 0xff) << 24) | (((x) & 0xff00) << 8) | (((x) >> 8) & 0xff00) | (((x) >> 24) & 0xff) )) +# define MP_BE32TOH(x) MP_HTOBE32(x) +#else +# define MP_HTOBE32(x) (x) +# define MP_BE32TOH(x) (x) +#endif +#endif + #endif // MICROPY_INCLUDED_PY_MPCONFIG_H diff --git a/python/src/py/mperrno.h b/python/src/py/mperrno.h index f439f6555..0cad75a17 100644 --- a/python/src/py/mperrno.h +++ b/python/src/py/mperrno.h @@ -122,7 +122,7 @@ #define MP_EPIPE EPIPE #define MP_EDOM EDOM #define MP_ERANGE ERANGE -#define MP_EWOULDBLOCK EAGAIN +#define MP_EWOULDBLOCK EWOULDBLOCK #define MP_EOPNOTSUPP EOPNOTSUPP #define MP_EAFNOSUPPORT EAFNOSUPPORT #define MP_EADDRINUSE EADDRINUSE diff --git a/python/src/py/mpprint.c b/python/src/py/mpprint.c index a569ef793..c2e65301c 100644 --- a/python/src/py/mpprint.c +++ b/python/src/py/mpprint.c @@ -446,11 +446,16 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { } } - // parse long specifiers (current not used) - //bool long_arg = false; + // parse long specifiers (only for LP64 model where they make a difference) + #ifndef __LP64__ + const + #endif + bool long_arg = false; if (*fmt == 'l') { ++fmt; - //long_arg = true; + #ifdef __LP64__ + long_arg = true; + #endif } if (*fmt == '\0') { @@ -505,14 +510,21 @@ int mp_vprintf(const mp_print_t *print, const char *fmt, va_list args) { chrs += mp_print_int(print, va_arg(args, int), 1, 10, 'a', flags, fill, width); break; case 'x': - chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 16, 'a', flags, fill, width); - break; - case 'X': - chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 16, 'A', flags, fill, width); + case 'X': { + char fmt_c = *fmt - 'X' + 'A'; + mp_uint_t val; + if (long_arg) { + val = va_arg(args, unsigned long int); + } else { + val = va_arg(args, unsigned int); + } + chrs += mp_print_int(print, val, 0, 16, fmt_c, flags, fill, width); break; + } case 'p': case 'P': // don't bother to handle upcase for 'P' - chrs += mp_print_int(print, va_arg(args, unsigned int), 0, 16, 'a', flags, fill, width); + // Use unsigned long int to work on both ILP32 and LP64 systems + chrs += mp_print_int(print, va_arg(args, unsigned long int), 0, 16, 'a', flags, fill, width); break; #if MICROPY_PY_BUILTINS_FLOAT case 'e': diff --git a/python/src/py/mpstate.h b/python/src/py/mpstate.h index 6a39ebdea..16c4bf6c5 100644 --- a/python/src/py/mpstate.h +++ b/python/src/py/mpstate.h @@ -78,7 +78,6 @@ typedef struct _mp_state_mem_t { int gc_stack_overflow; size_t gc_stack[MICROPY_ALLOC_GC_STACK_SIZE]; - size_t *gc_sp; uint16_t gc_lock_depth; // This variable controls auto garbage collection. If set to 0 then the @@ -167,6 +166,10 @@ typedef struct _mp_state_vm_t { // root pointers for extmod + #if MICROPY_REPL_EVENT_DRIVEN + vstr_t *repl_line; + #endif + #if MICROPY_PY_OS_DUPTERM mp_obj_t dupterm_objs[MICROPY_PY_OS_DUPTERM]; mp_obj_t dupterm_arr_obj; @@ -196,7 +199,9 @@ typedef struct _mp_state_vm_t { mp_thread_mutex_t qstr_mutex; #endif + #if MICROPY_ENABLE_COMPILER mp_uint_t mp_optimise_value; + #endif // size of the emergency exception buf, if it's dynamically allocated #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF && MICROPY_EMERGENCY_EXCEPTION_BUF_SIZE == 0 @@ -215,7 +220,6 @@ typedef struct _mp_state_thread_t { mp_obj_dict_t *dict_locals; mp_obj_dict_t *dict_globals; - // Note: nlr asm code has the offset of this hard-coded nlr_buf_t *nlr_top; // ROOT POINTER // Stack top at the start of program @@ -224,6 +228,12 @@ typedef struct _mp_state_thread_t { #if MICROPY_STACK_CHECK size_t stack_limit; #endif + + #if MICROPY_ENABLE_PYSTACK + uint8_t *pystack_start; + uint8_t *pystack_end; + uint8_t *pystack_cur; + #endif } mp_state_thread_t; // This structure combines the above 3 structures. diff --git a/python/src/py/mpz.c b/python/src/py/mpz.c index d300a8e5d..fa5086862 100644 --- a/python/src/py/mpz.c +++ b/python/src/py/mpz.c @@ -537,83 +537,57 @@ STATIC void mpn_div(mpz_dig_t *num_dig, size_t *num_len, const mpz_dig_t *den_di // not to overflow the borrow variable. And the shifting of // borrow needs some special logic (it's a shift right with // round up). - - if (DIG_SIZE < 8 * sizeof(mpz_dbl_dig_t) / 2) { - const mpz_dig_t *d = den_dig; - mpz_dbl_dig_t d_norm = 0; - mpz_dbl_dig_signed_t borrow = 0; - - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - borrow += (mpz_dbl_dig_t)*n - (mpz_dbl_dig_t)quo * (d_norm & DIG_MASK); // will overflow if DIG_SIZE >= 8*sizeof(mpz_dbl_dig_t)/2 - *n = borrow & DIG_MASK; - borrow >>= DIG_SIZE; - } - borrow += *num_dig; // will overflow if DIG_SIZE >= 8*sizeof(mpz_dbl_dig_t)/2 - *num_dig = borrow & DIG_MASK; - borrow >>= DIG_SIZE; - - // adjust quotient if it is too big - for (; borrow != 0; --quo) { - d = den_dig; - d_norm = 0; - mpz_dbl_dig_t carry = 0; - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - carry += (mpz_dbl_dig_t)*n + (d_norm & DIG_MASK); - *n = carry & DIG_MASK; - carry >>= DIG_SIZE; - } - carry += *num_dig; - *num_dig = carry & DIG_MASK; - carry >>= DIG_SIZE; - - borrow += carry; - } - } else { // DIG_SIZE == 8 * sizeof(mpz_dbl_dig_t) / 2 - const mpz_dig_t *d = den_dig; - mpz_dbl_dig_t d_norm = 0; - mpz_dbl_dig_t borrow = 0; - - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - mpz_dbl_dig_t x = (mpz_dbl_dig_t)quo * (d_norm & DIG_MASK); - if (x >= *n || *n - x <= borrow) { - borrow += (mpz_dbl_dig_t)x - (mpz_dbl_dig_t)*n; - *n = (-borrow) & DIG_MASK; - borrow = (borrow >> DIG_SIZE) + ((borrow & DIG_MASK) == 0 ? 0 : 1); // shift-right with round-up - } else { - *n = ((mpz_dbl_dig_t)*n - (mpz_dbl_dig_t)x - (mpz_dbl_dig_t)borrow) & DIG_MASK; - borrow = 0; - } - } - if (borrow >= *num_dig) { - borrow -= (mpz_dbl_dig_t)*num_dig; - *num_dig = (-borrow) & DIG_MASK; + // + const mpz_dig_t *d = den_dig; + mpz_dbl_dig_t d_norm = 0; + mpz_dbl_dig_t borrow = 0; + for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { + d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); + mpz_dbl_dig_t x = (mpz_dbl_dig_t)quo * (d_norm & DIG_MASK); + #if DIG_SIZE < MPZ_DBL_DIG_SIZE / 2 + borrow += (mpz_dbl_dig_t)*n - x; // will overflow if DIG_SIZE >= MPZ_DBL_DIG_SIZE/2 + *n = borrow & DIG_MASK; + borrow = (mpz_dbl_dig_signed_t)borrow >> DIG_SIZE; + #else // DIG_SIZE == MPZ_DBL_DIG_SIZE / 2 + if (x >= *n || *n - x <= borrow) { + borrow += x - (mpz_dbl_dig_t)*n; + *n = (-borrow) & DIG_MASK; borrow = (borrow >> DIG_SIZE) + ((borrow & DIG_MASK) == 0 ? 0 : 1); // shift-right with round-up } else { - *num_dig = (*num_dig - borrow) & DIG_MASK; + *n = ((mpz_dbl_dig_t)*n - x - borrow) & DIG_MASK; borrow = 0; } + #endif + } - // adjust quotient if it is too big - for (; borrow != 0; --quo) { - d = den_dig; - d_norm = 0; - mpz_dbl_dig_t carry = 0; - for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { - d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); - carry += (mpz_dbl_dig_t)*n + (d_norm & DIG_MASK); - *n = carry & DIG_MASK; - carry >>= DIG_SIZE; - } - carry += (mpz_dbl_dig_t)*num_dig; - *num_dig = carry & DIG_MASK; + #if DIG_SIZE < MPZ_DBL_DIG_SIZE / 2 + // Borrow was negative in the above for-loop, make it positive for next if-block. + borrow = -borrow; + #endif + + // At this point we have either: + // + // 1. quo was the correct value and the most-sig-digit of num is exactly + // cancelled by borrow (borrow == *num_dig). In this case there is + // nothing more to do. + // + // 2. quo was too large, we subtracted too many den from num, and the + // most-sig-digit of num is 1 less than borrow (borrow == *num_dig + 1). + // In this case we must reduce quo and add back den to num until the + // carry from this operation cancels out the borrow. + // + borrow -= *num_dig; + for (; borrow != 0; --quo) { + d = den_dig; + d_norm = 0; + mpz_dbl_dig_t carry = 0; + for (mpz_dig_t *n = num_dig - den_len; n < num_dig; ++n, ++d) { + d_norm = ((mpz_dbl_dig_t)*d << norm_shift) | (d_norm >> DIG_SIZE); + carry += (mpz_dbl_dig_t)*n + (d_norm & DIG_MASK); + *n = carry & DIG_MASK; carry >>= DIG_SIZE; - - //assert(borrow >= carry); // enable this to check the logic - borrow -= carry; } + borrow -= carry; } // store this digit of the quotient @@ -731,17 +705,14 @@ STATIC void mpz_need_dig(mpz_t *z, size_t need) { } STATIC mpz_t *mpz_clone(const mpz_t *src) { + assert(src->alloc != 0); mpz_t *z = m_new_obj(mpz_t); z->neg = src->neg; z->fixed_dig = 0; z->alloc = src->alloc; z->len = src->len; - if (src->dig == NULL) { - z->dig = NULL; - } else { - z->dig = m_new(mpz_dig_t, z->alloc); - memcpy(z->dig, src->dig, src->alloc * sizeof(mpz_dig_t)); - } + z->dig = m_new(mpz_dig_t, z->alloc); + memcpy(z->dig, src->dig, src->alloc * sizeof(mpz_dig_t)); return z; } @@ -1009,6 +980,7 @@ these functions are unused /* returns abs(z) */ mpz_t *mpz_abs(const mpz_t *z) { + // TODO: handle case of z->alloc=0 mpz_t *z2 = mpz_clone(z); z2->neg = 0; return z2; @@ -1017,6 +989,7 @@ mpz_t *mpz_abs(const mpz_t *z) { /* returns -z */ mpz_t *mpz_neg(const mpz_t *z) { + // TODO: handle case of z->alloc=0 mpz_t *z2 = mpz_clone(z); z2->neg = 1 - z2->neg; return z2; @@ -1390,13 +1363,14 @@ void mpz_pow_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs) { can have dest, lhs, rhs the same; mod can't be the same as dest */ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t *mod) { - if (lhs->len == 0 || rhs->neg != 0) { + if (lhs->len == 0 || rhs->neg != 0 || (mod->len == 1 && mod->dig[0] == 1)) { mpz_set_from_int(dest, 0); return; } + mpz_set_from_int(dest, 1); + if (rhs->len == 0) { - mpz_set_from_int(dest, 1); return; } @@ -1404,8 +1378,6 @@ void mpz_pow3_inpl(mpz_t *dest, const mpz_t *lhs, const mpz_t *rhs, const mpz_t mpz_t *n = mpz_clone(rhs); mpz_t quo; mpz_init_zero(&quo); - mpz_set_from_int(dest, 1); - while (n->len > 0) { if ((n->dig[0] & 1) != 0) { mpz_mul_inpl(dest, dest, x); @@ -1435,6 +1407,7 @@ these functions are unused */ mpz_t *mpz_gcd(const mpz_t *z1, const mpz_t *z2) { if (z1->len == 0) { + // TODO: handle case of z2->alloc=0 mpz_t *a = mpz_clone(z2); a->neg = 0; return a; @@ -1674,16 +1647,12 @@ char *mpz_as_str(const mpz_t *i, unsigned int base) { } #endif -// assumes enough space as calculated by mp_int_format_size +// assumes enough space in str as calculated by mp_int_format_size +// base must be between 2 and 32 inclusive // returns length of string, not including null byte size_t mpz_as_str_inpl(const mpz_t *i, unsigned int base, const char *prefix, char base_char, char comma, char *str) { - if (str == NULL) { - return 0; - } - if (base < 2 || base > 32) { - str[0] = 0; - return 0; - } + assert(str != NULL); + assert(2 <= base && base <= 32); size_t ilen = i->len; diff --git a/python/src/py/mpz.h b/python/src/py/mpz.h index e2d0c30aa..3c36cac66 100644 --- a/python/src/py/mpz.h +++ b/python/src/py/mpz.h @@ -55,18 +55,22 @@ #endif #if MPZ_DIG_SIZE > 16 +#define MPZ_DBL_DIG_SIZE (64) typedef uint32_t mpz_dig_t; typedef uint64_t mpz_dbl_dig_t; typedef int64_t mpz_dbl_dig_signed_t; #elif MPZ_DIG_SIZE > 8 +#define MPZ_DBL_DIG_SIZE (32) typedef uint16_t mpz_dig_t; typedef uint32_t mpz_dbl_dig_t; typedef int32_t mpz_dbl_dig_signed_t; #elif MPZ_DIG_SIZE > 4 +#define MPZ_DBL_DIG_SIZE (16) typedef uint8_t mpz_dig_t; typedef uint16_t mpz_dbl_dig_t; typedef int16_t mpz_dbl_dig_signed_t; #else +#define MPZ_DBL_DIG_SIZE (8) typedef uint8_t mpz_dig_t; typedef uint8_t mpz_dbl_dig_t; typedef int8_t mpz_dbl_dig_signed_t; diff --git a/python/src/py/nlr.c b/python/src/py/nlr.c new file mode 100644 index 000000000..03d01577e --- /dev/null +++ b/python/src/py/nlr.c @@ -0,0 +1,51 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "py/mpstate.h" + +#if !MICROPY_NLR_SETJMP +// When not using setjmp, nlr_push_tail is called from inline asm so needs special care +#if MICROPY_NLR_X86 && MICROPY_NLR_OS_WINDOWS +// On these 32-bit platforms make sure nlr_push_tail doesn't have a leading underscore +unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); +#else +// LTO can't see inside inline asm functions so explicitly mark nlr_push_tail as used +__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); +#endif +#endif + +unsigned int nlr_push_tail(nlr_buf_t *nlr) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + nlr->prev = *top; + MP_NLR_SAVE_PYSTACK(nlr); + *top = nlr; + return 0; // normal return +} + +void nlr_pop(void) { + nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); + *top = (*top)->prev; +} diff --git a/python/src/py/nlr.h b/python/src/py/nlr.h index 63fe392d9..90595a12d 100644 --- a/python/src/py/nlr.h +++ b/python/src/py/nlr.h @@ -30,29 +30,34 @@ // exception handling, basically a stack of setjmp/longjmp buffers #include -#include #include #include "py/mpconfig.h" -typedef struct _nlr_buf_t nlr_buf_t; -struct _nlr_buf_t { - // the entries here must all be machine word size - nlr_buf_t *prev; - void *ret_val; // always a concrete object (an exception instance) -#if !defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP +// If MICROPY_NLR_SETJMP is not enabled then auto-detect the machine arch +#if !MICROPY_NLR_SETJMP +// A lot of nlr-related things need different treatment on Windows +#if defined(_WIN32) || defined(__CYGWIN__) +#define MICROPY_NLR_OS_WINDOWS 1 +#else +#define MICROPY_NLR_OS_WINDOWS 0 +#endif #if defined(__i386__) - void *regs[6]; + #define MICROPY_NLR_X86 (1) + #define MICROPY_NLR_NUM_REGS (6) #elif defined(__x86_64__) - #if defined(__CYGWIN__) - void *regs[12]; - #else - void *regs[8]; - #endif + #define MICROPY_NLR_X64 (1) + #if MICROPY_NLR_OS_WINDOWS + #define MICROPY_NLR_NUM_REGS (10) + #else + #define MICROPY_NLR_NUM_REGS (8) + #endif #elif defined(__thumb2__) || defined(__thumb__) || defined(__arm__) - void *regs[10]; + #define MICROPY_NLR_THUMB (1) + #define MICROPY_NLR_NUM_REGS (10) #elif defined(__xtensa__) - void *regs[10]; + #define MICROPY_NLR_XTENSA (1) + #define MICROPY_NLR_NUM_REGS (10) #else #define MICROPY_NLR_SETJMP (1) //#warning "No native NLR support for this arch, using setjmp implementation" @@ -60,24 +65,58 @@ struct _nlr_buf_t { #endif #if MICROPY_NLR_SETJMP - jmp_buf jmpbuf; +#include #endif + +typedef struct _nlr_buf_t nlr_buf_t; +struct _nlr_buf_t { + // the entries here must all be machine word size + nlr_buf_t *prev; + void *ret_val; // always a concrete object (an exception instance) + + #if MICROPY_NLR_SETJMP + jmp_buf jmpbuf; + #else + void *regs[MICROPY_NLR_NUM_REGS]; + #endif + + #if MICROPY_ENABLE_PYSTACK + void *pystack; + #endif }; -#if MICROPY_NLR_SETJMP -#include "py/mpstate.h" +// Helper macros to save/restore the pystack state +#if MICROPY_ENABLE_PYSTACK +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (nlr_buf)->pystack = MP_STATE_THREAD(pystack_cur) +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) MP_STATE_THREAD(pystack_cur) = (nlr_buf)->pystack +#else +#define MP_NLR_SAVE_PYSTACK(nlr_buf) (void)nlr_buf +#define MP_NLR_RESTORE_PYSTACK(nlr_buf) (void)nlr_buf +#endif -NORETURN void nlr_setjmp_jump(void *val); +// Helper macro to use at the start of a specific nlr_jump implementation +#define MP_NLR_JUMP_HEAD(val, top) \ + nlr_buf_t **_top_ptr = &MP_STATE_THREAD(nlr_top); \ + nlr_buf_t *top = *_top_ptr; \ + if (top == NULL) { \ + nlr_jump_fail(val); \ + } \ + top->ret_val = val; \ + MP_NLR_RESTORE_PYSTACK(top); \ + *_top_ptr = top->prev; \ + +#if MICROPY_NLR_SETJMP // nlr_push() must be defined as a macro, because "The stack context will be // invalidated if the function which called setjmp() returns." -#define nlr_push(buf) ((buf)->prev = MP_STATE_THREAD(nlr_top), MP_STATE_THREAD(nlr_top) = (buf), setjmp((buf)->jmpbuf)) -#define nlr_pop() { MP_STATE_THREAD(nlr_top) = MP_STATE_THREAD(nlr_top)->prev; } -#define nlr_jump(val) nlr_setjmp_jump(val) +// For this case it is safe to call nlr_push_tail() first. +#define nlr_push(buf) (nlr_push_tail(buf), setjmp((buf)->jmpbuf)) #else unsigned int nlr_push(nlr_buf_t *); +#endif + +unsigned int nlr_push_tail(nlr_buf_t *top); void nlr_pop(void); NORETURN void nlr_jump(void *val); -#endif // This must be implemented by a port. It's called by nlr_jump // if no nlr buf has been pushed. It must not return, but rather @@ -106,7 +145,6 @@ NORETURN void nlr_jump_fail(void *val); /* #define nlr_push(val) \ printf("nlr_push: before: nlr_top=%p, val=%p\n", MP_STATE_THREAD(nlr_top), val),assert(MP_STATE_THREAD(nlr_top) != val),nlr_push(val) -#endif */ #endif diff --git a/python/src/py/nlrsetjmp.c b/python/src/py/nlrsetjmp.c index 1fb459440..960dd86f5 100644 --- a/python/src/py/nlrsetjmp.c +++ b/python/src/py/nlrsetjmp.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013, 2014 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,17 +24,18 @@ * THE SOFTWARE. */ -#include "py/nlr.h" +#include "py/mpstate.h" #if MICROPY_NLR_SETJMP -void nlr_setjmp_jump(void *val) { +void nlr_jump(void *val) { nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); nlr_buf_t *top = *top_ptr; if (top == NULL) { nlr_jump_fail(val); } top->ret_val = val; + MP_NLR_RESTORE_PYSTACK(top); *top_ptr = top->prev; longjmp(top->jmpbuf, 1); } diff --git a/python/src/py/nlrthumb.c b/python/src/py/nlrthumb.c index 6e7d71766..c28302355 100644 --- a/python/src/py/nlrthumb.c +++ b/python/src/py/nlrthumb.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013-2016 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) +#if MICROPY_NLR_THUMB #undef nlr_push @@ -76,30 +76,15 @@ __attribute__((naked)) unsigned int nlr_push(nlr_buf_t *nlr) { #endif ); - return 0; // needed to silence compiler warning + #if !defined(__clang__) && defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)) + // Older versions of gcc give an error when naked functions don't return a value + // Additionally exclude Clang as it also defines __GNUC__ but doesn't need this statement + return 0; + #endif } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - -NORETURN __attribute__((naked)) void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - *top_ptr = top->prev; +NORETURN void nlr_jump(void *val) { + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "mov r0, %0 \n" // r0 points to nlr_buf @@ -136,7 +121,11 @@ NORETURN __attribute__((naked)) void nlr_jump(void *val) { : // clobbered registers ); + #if defined(__GNUC__) + __builtin_unreachable(); + #else for (;;); // needed to silence compiler warning + #endif } -#endif // (!defined(MICROPY_NLR_SETJMP) || !MICROPY_NLR_SETJMP) && (defined(__thumb2__) || defined(__thumb__) || defined(__arm__)) +#endif // MICROPY_NLR_THUMB diff --git a/python/src/py/nlrx64.c b/python/src/py/nlrx64.c index 847d10398..a3a1cf341 100644 --- a/python/src/py/nlrx64.c +++ b/python/src/py/nlrx64.c @@ -26,25 +26,19 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__x86_64__) +#if MICROPY_NLR_X64 #undef nlr_push // x86-64 callee-save registers are: // rbx, rbp, rsp, r12, r13, r14, r15 -#if defined(_WIN32) || defined(__CYGWIN__) -#define NLR_OS_WINDOWS 1 -#else -#define NLR_OS_WINDOWS 0 -#endif - __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); unsigned int nlr_push(nlr_buf_t *nlr) { (void)nlr; - #if NLR_OS_WINDOWS + #if MICROPY_NLR_OS_WINDOWS __asm volatile ( "movq (%rsp), %rax \n" // load return %rip @@ -88,31 +82,12 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "movq %0, %%rcx \n" // %rcx points to nlr_buf - #if NLR_OS_WINDOWS + #if MICROPY_NLR_OS_WINDOWS "movq 88(%%rcx), %%rsi \n" // load saved %rsi "movq 80(%%rcx), %%rdi \n" // load saved %rdr #endif @@ -136,4 +111,4 @@ NORETURN void nlr_jump(void *val) { for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__x86_64__) +#endif // MICROPY_NLR_X64 diff --git a/python/src/py/nlrx86.c b/python/src/py/nlrx86.c index 094dea3cc..23882cc30 100644 --- a/python/src/py/nlrx86.c +++ b/python/src/py/nlrx86.c @@ -26,20 +26,14 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__i386__) +#if MICROPY_NLR_X86 #undef nlr_push // For reference, x86 callee save regs are: // ebx, esi, edi, ebp, esp, eip -#if defined(_WIN32) || defined(__CYGWIN__) -#define NLR_OS_WINDOWS 1 -#else -#define NLR_OS_WINDOWS 0 -#endif - -#if NLR_OS_WINDOWS +#if MICROPY_NLR_OS_WINDOWS unsigned int nlr_push_tail(nlr_buf_t *nlr) asm("nlr_push_tail"); #else __attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr); @@ -70,27 +64,8 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "mov %0, %%edx \n" // %edx points to nlr_buf @@ -112,4 +87,4 @@ NORETURN void nlr_jump(void *val) { for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__i386__) +#endif // MICROPY_NLR_X86 diff --git a/python/src/py/nlrxtensa.c b/python/src/py/nlrxtensa.c index 4520e7e7a..cd3dee364 100644 --- a/python/src/py/nlrxtensa.c +++ b/python/src/py/nlrxtensa.c @@ -26,7 +26,7 @@ #include "py/mpstate.h" -#if !MICROPY_NLR_SETJMP && defined(__xtensa__) +#if MICROPY_NLR_XTENSA #undef nlr_push @@ -55,27 +55,8 @@ unsigned int nlr_push(nlr_buf_t *nlr) { return 0; // needed to silence compiler warning } -__attribute__((used)) unsigned int nlr_push_tail(nlr_buf_t *nlr) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - nlr->prev = *top; - *top = nlr; - return 0; // normal return -} - -void nlr_pop(void) { - nlr_buf_t **top = &MP_STATE_THREAD(nlr_top); - *top = (*top)->prev; -} - NORETURN void nlr_jump(void *val) { - nlr_buf_t **top_ptr = &MP_STATE_THREAD(nlr_top); - nlr_buf_t *top = *top_ptr; - if (top == NULL) { - nlr_jump_fail(val); - } - - top->ret_val = val; - *top_ptr = top->prev; + MP_NLR_JUMP_HEAD(val, top) __asm volatile ( "mov.n a2, %0 \n" // a2 points to nlr_buf @@ -99,4 +80,4 @@ NORETURN void nlr_jump(void *val) { for (;;); // needed to silence compiler warning } -#endif // !MICROPY_NLR_SETJMP && defined(__xtensa__) +#endif // MICROPY_NLR_XTENSA diff --git a/python/src/py/obj.h b/python/src/py/obj.h index 77f0f2298..95a94836e 100644 --- a/python/src/py/obj.h +++ b/python/src/py/obj.h @@ -170,8 +170,8 @@ static inline bool MP_OBJ_IS_OBJ(mp_const_obj_t o) static inline bool MP_OBJ_IS_SMALL_INT(mp_const_obj_t o) { return ((((mp_int_t)(o)) & 0xffff000000000000) == 0x0001000000000000); } -#define MP_OBJ_SMALL_INT_VALUE(o) (((intptr_t)(o)) >> 1) -#define MP_OBJ_NEW_SMALL_INT(small_int) ((mp_obj_t)(((uintptr_t)(small_int)) << 1) | 0x0001000000000001) +#define MP_OBJ_SMALL_INT_VALUE(o) (((mp_int_t)((o) << 16)) >> 17) +#define MP_OBJ_NEW_SMALL_INT(small_int) (((((uint64_t)(small_int)) & 0x7fffffffffff) << 1) | 0x0001000000000001) static inline bool MP_OBJ_IS_QSTR(mp_const_obj_t o) { return ((((mp_int_t)(o)) & 0xffff000000000000) == 0x0002000000000000); } @@ -179,7 +179,7 @@ static inline bool MP_OBJ_IS_QSTR(mp_const_obj_t o) #define MP_OBJ_NEW_QSTR(qst) ((mp_obj_t)((((mp_uint_t)(qst)) << 1) | 0x0002000000000001)) #if MICROPY_PY_BUILTINS_FLOAT -#define mp_const_float_e {((mp_obj_t)((uint64_t)0x4005bf0a8b125769 + 0x8004000000000000))} +#define mp_const_float_e {((mp_obj_t)((uint64_t)0x4005bf0a8b145769 + 0x8004000000000000))} #define mp_const_float_pi {((mp_obj_t)((uint64_t)0x400921fb54442d18 + 0x8004000000000000))} static inline bool mp_obj_is_float(mp_const_obj_t o) { @@ -250,6 +250,8 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; // The macros below are derived from the ones above and are used to // check for more specific object types. +// Note: these are kept as macros because inline functions sometimes use much +// more code space than the equivalent macros, depending on the compiler. #define MP_OBJ_IS_TYPE(o, t) (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type == (t))) // this does not work for checking int, str or fun; use below macros for that #define MP_OBJ_IS_INT(o) (MP_OBJ_IS_SMALL_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_int)) @@ -257,17 +259,6 @@ typedef struct _mp_rom_obj_t { mp_const_obj_t o; } mp_rom_obj_t; #define MP_OBJ_IS_STR_OR_BYTES(o) (MP_OBJ_IS_QSTR(o) || (MP_OBJ_IS_OBJ(o) && ((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type->binary_op == mp_obj_str_binary_op)) #define MP_OBJ_IS_FUN(o) (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)MP_OBJ_TO_PTR(o))->type->name == MP_QSTR_function)) -// Note: inline functions sometimes use much more code space than the -// equivalent macros, depending on the compiler. -//static inline bool MP_OBJ_IS_TYPE(mp_const_obj_t o, const mp_obj_type_t *t) { return (MP_OBJ_IS_OBJ(o) && (((mp_obj_base_t*)(o))->type == (t))); } // this does not work for checking a string, use below macro for that -//static inline bool MP_OBJ_IS_INT(mp_const_obj_t o) { return (MP_OBJ_IS_SMALL_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_int)); } // returns true if o is a small int or long int -// Need to forward declare these for the inline function to compile. -extern const mp_obj_type_t mp_type_int; -extern const mp_obj_type_t mp_type_bool; -static inline bool mp_obj_is_integer(mp_const_obj_t o) { return MP_OBJ_IS_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_bool); } // returns true if o is bool, small int or long int -//static inline bool MP_OBJ_IS_STR(mp_const_obj_t o) { return (MP_OBJ_IS_QSTR(o) || MP_OBJ_IS_TYPE(o, &mp_type_str)); } - - // These macros are used to declare and define constant function objects // You can put "static" in front of the definitions to make them local @@ -553,6 +544,7 @@ extern const mp_obj_type_t mp_type_list; extern const mp_obj_type_t mp_type_map; // map (the python builtin, not the dict implementation detail) extern const mp_obj_type_t mp_type_enumerate; extern const mp_obj_type_t mp_type_filter; +extern const mp_obj_type_t mp_type_deque; extern const mp_obj_type_t mp_type_dict; extern const mp_obj_type_t mp_type_ordereddict; extern const mp_obj_type_t mp_type_range; @@ -624,7 +616,6 @@ extern const struct _mp_obj_str_t mp_const_empty_bytes_obj; extern const struct _mp_obj_tuple_t mp_const_empty_tuple_obj; extern const struct _mp_obj_singleton_t mp_const_ellipsis_obj; extern const struct _mp_obj_singleton_t mp_const_notimplemented_obj; -extern const struct _mp_obj_exception_t mp_const_MemoryError_obj; extern const struct _mp_obj_exception_t mp_const_GeneratorExit_obj; // General API for objects @@ -637,7 +628,8 @@ mp_obj_t mp_obj_new_int_from_uint(mp_uint_t value); mp_obj_t mp_obj_new_int_from_str_len(const char **str, size_t len, bool neg, unsigned int base); mp_obj_t mp_obj_new_int_from_ll(long long val); // this must return a multi-precision integer object (or raise an overflow exception) mp_obj_t mp_obj_new_int_from_ull(unsigned long long val); // this must return a multi-precision integer object (or raise an overflow exception) -mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already); +mp_obj_t mp_obj_new_str(const char* data, size_t len); +mp_obj_t mp_obj_new_str_via_qstr(const char* data, size_t len); mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr); mp_obj_t mp_obj_new_bytes(const byte* data, size_t len); mp_obj_t mp_obj_new_bytearray(size_t n, void *items); @@ -680,6 +672,7 @@ bool mp_obj_is_true(mp_obj_t arg); bool mp_obj_is_callable(mp_obj_t o_in); bool mp_obj_equal(mp_obj_t o1, mp_obj_t o2); +static inline bool mp_obj_is_integer(mp_const_obj_t o) { return MP_OBJ_IS_INT(o) || MP_OBJ_IS_TYPE(o, &mp_type_bool); } // returns true if o is bool, small int or long int mp_int_t mp_obj_get_int(mp_const_obj_t arg); mp_int_t mp_obj_get_int_truncated(mp_const_obj_t arg); bool mp_obj_get_int_maybe(mp_const_obj_t arg, mp_int_t *value); @@ -727,6 +720,7 @@ qstr mp_obj_str_get_qstr(mp_obj_t self_in); // use this if you will anyway conve const char *mp_obj_str_get_str(mp_obj_t self_in); // use this only if you need the string to be null terminated const char *mp_obj_str_get_data(mp_obj_t self_in, size_t *len); mp_obj_t mp_obj_str_intern(mp_obj_t str); +mp_obj_t mp_obj_str_intern_checked(mp_obj_t obj); void mp_str_print_quoted(const mp_print_t *print, const byte *str_data, size_t str_len, bool is_bytes); #if MICROPY_PY_BUILTINS_FLOAT @@ -751,8 +745,6 @@ void mp_obj_tuple_del(mp_obj_t self_in); mp_int_t mp_obj_tuple_hash(mp_obj_t self_in); // list -struct _mp_obj_list_t; -void mp_obj_list_init(struct _mp_obj_list_t *o, size_t n); mp_obj_t mp_obj_list_append(mp_obj_t self_in, mp_obj_t arg); mp_obj_t mp_obj_list_remove(mp_obj_t self_in, mp_obj_t value); void mp_obj_list_get(mp_obj_t self_in, size_t *len, mp_obj_t **items); diff --git a/python/src/py/objarray.c b/python/src/py/objarray.c index 7003ec9e7..a35539484 100644 --- a/python/src/py/objarray.c +++ b/python/src/py/objarray.c @@ -269,8 +269,7 @@ STATIC mp_obj_t array_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs return lhs_in; } - case MP_BINARY_OP_IN: { - /* NOTE `a in b` is `b.__contains__(a)` */ + case MP_BINARY_OP_CONTAINS: { mp_buffer_info_t lhs_bufinfo; mp_buffer_info_t rhs_bufinfo; diff --git a/python/src/py/objboundmeth.c b/python/src/py/objboundmeth.c index 890f8b15b..b0df6a68a 100644 --- a/python/src/py/objboundmeth.c +++ b/python/src/py/objboundmeth.c @@ -51,6 +51,9 @@ mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, s // need to insert self before all other args and then call meth size_t n_total = n_args + 2 * n_kw; mp_obj_t *args2 = NULL; + #if MICROPY_ENABLE_PYSTACK + args2 = mp_pystack_alloc(sizeof(mp_obj_t) * (1 + n_total)); + #else mp_obj_t *free_args2 = NULL; if (n_total > 4) { // try to use heap to allocate temporary args array @@ -61,12 +64,17 @@ mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, s // (fallback to) use stack to allocate temporary args array args2 = alloca(sizeof(mp_obj_t) * (1 + n_total)); } + #endif args2[0] = self; memcpy(args2 + 1, args, n_total * sizeof(mp_obj_t)); mp_obj_t res = mp_call_function_n_kw(meth, n_args + 1, n_kw, args2); + #if MICROPY_ENABLE_PYSTACK + mp_pystack_free(args2); + #else if (free_args2 != NULL) { m_del(mp_obj_t, free_args2, 1 + n_total); } + #endif return res; } diff --git a/python/src/py/objdeque.c b/python/src/py/objdeque.c new file mode 100644 index 000000000..1cff1f8d3 --- /dev/null +++ b/python/src/py/objdeque.c @@ -0,0 +1,167 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2018 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include // for ssize_t +#include + +#include "py/mpconfig.h" +#if MICROPY_PY_COLLECTIONS_DEQUE + +#include "py/runtime.h" + +typedef struct _mp_obj_deque_t { + mp_obj_base_t base; + size_t alloc; + size_t i_get; + size_t i_put; + mp_obj_t *items; + uint32_t flags; + #define FLAG_CHECK_OVERFLOW 1 +} mp_obj_deque_t; + +STATIC mp_obj_t deque_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) { + mp_arg_check_num(n_args, n_kw, 2, 3, false); + + /* Initialization from existing sequence is not supported, so an empty + tuple must be passed as such. */ + if (args[0] != mp_const_empty_tuple) { + mp_raise_ValueError(NULL); + } + + // Protect against -1 leading to zero-length allocation and bad array access + mp_int_t maxlen = mp_obj_get_int(args[1]); + if (maxlen < 0) { + mp_raise_ValueError(NULL); + } + + mp_obj_deque_t *o = m_new_obj(mp_obj_deque_t); + o->base.type = type; + o->alloc = maxlen + 1; + o->i_get = o->i_put = 0; + o->items = m_new0(mp_obj_t, o->alloc); + + if (n_args > 2) { + o->flags = mp_obj_get_int(args[2]); + } + + return MP_OBJ_FROM_PTR(o); +} + +STATIC mp_obj_t deque_unary_op(mp_unary_op_t op, mp_obj_t self_in) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + switch (op) { + case MP_UNARY_OP_BOOL: + return mp_obj_new_bool(self->i_get != self->i_put); + case MP_UNARY_OP_LEN: { + ssize_t len = self->i_put - self->i_get; + if (len < 0) { + len += self->alloc; + } + return MP_OBJ_NEW_SMALL_INT(len); + } + #if MICROPY_PY_SYS_GETSIZEOF + case MP_UNARY_OP_SIZEOF: { + size_t sz = sizeof(*self) + sizeof(mp_obj_t) * self->alloc; + return MP_OBJ_NEW_SMALL_INT(sz); + } + #endif + default: + return MP_OBJ_NULL; // op not supported + } +} + +STATIC mp_obj_t mp_obj_deque_append(mp_obj_t self_in, mp_obj_t arg) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + + size_t new_i_put = self->i_put + 1; + if (new_i_put == self->alloc) { + new_i_put = 0; + } + + if (self->flags & FLAG_CHECK_OVERFLOW && new_i_put == self->i_get) { + mp_raise_msg(&mp_type_IndexError, "full"); + } + + self->items[self->i_put] = arg; + self->i_put = new_i_put; + + if (self->i_get == new_i_put) { + if (++self->i_get == self->alloc) { + self->i_get = 0; + } + } + + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(deque_append_obj, mp_obj_deque_append); + +STATIC mp_obj_t deque_popleft(mp_obj_t self_in) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + + if (self->i_get == self->i_put) { + mp_raise_msg(&mp_type_IndexError, "empty"); + } + + mp_obj_t ret = self->items[self->i_get]; + self->items[self->i_get] = MP_OBJ_NULL; + + if (++self->i_get == self->alloc) { + self->i_get = 0; + } + + return ret; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(deque_popleft_obj, deque_popleft); + +#if 0 +STATIC mp_obj_t deque_clear(mp_obj_t self_in) { + mp_obj_deque_t *self = MP_OBJ_TO_PTR(self_in); + self->i_get = self->i_put = 0; + mp_seq_clear(self->items, 0, self->alloc, sizeof(*self->items)); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_1(deque_clear_obj, deque_clear); +#endif + +STATIC const mp_rom_map_elem_t deque_locals_dict_table[] = { + { MP_ROM_QSTR(MP_QSTR_append), MP_ROM_PTR(&deque_append_obj) }, + #if 0 + { MP_ROM_QSTR(MP_QSTR_clear), MP_ROM_PTR(&deque_clear_obj) }, + #endif + { MP_ROM_QSTR(MP_QSTR_popleft), MP_ROM_PTR(&deque_popleft_obj) }, +}; + +STATIC MP_DEFINE_CONST_DICT(deque_locals_dict, deque_locals_dict_table); + +const mp_obj_type_t mp_type_deque = { + { &mp_type_type }, + .name = MP_QSTR_deque, + .make_new = deque_make_new, + .unary_op = deque_unary_op, + .locals_dict = (mp_obj_dict_t*)&deque_locals_dict, +}; + +#endif // MICROPY_PY_COLLECTIONS_DEQUE diff --git a/python/src/py/objdict.c b/python/src/py/objdict.c index 1553a83b4..c0647067a 100644 --- a/python/src/py/objdict.c +++ b/python/src/py/objdict.c @@ -115,7 +115,7 @@ STATIC mp_obj_t dict_unary_op(mp_unary_op_t op, mp_obj_t self_in) { STATIC mp_obj_t dict_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { mp_obj_dict_t *o = MP_OBJ_TO_PTR(lhs_in); switch (op) { - case MP_BINARY_OP_IN: { + case MP_BINARY_OP_CONTAINS: { mp_map_elem_t *elem = mp_map_lookup(&o->map, rhs_in, MP_MAP_LOOKUP); return mp_obj_new_bool(elem != NULL); } @@ -193,42 +193,18 @@ STATIC mp_obj_t dict_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { } /******************************************************************************/ -/* dict iterator */ +/* dict methods */ -typedef struct _mp_obj_dict_it_t { - mp_obj_base_t base; - mp_fun_1_t iternext; - mp_obj_t dict; - size_t cur; -} mp_obj_dict_it_t; - -STATIC mp_obj_t dict_it_iternext(mp_obj_t self_in) { - mp_obj_dict_it_t *self = MP_OBJ_TO_PTR(self_in); - mp_map_elem_t *next = dict_iter_next(MP_OBJ_TO_PTR(self->dict), &self->cur); - - if (next == NULL) { - return MP_OBJ_STOP_ITERATION; - } else { - return next->key; +STATIC void mp_ensure_not_fixed(const mp_obj_dict_t *dict) { + if (dict->map.is_fixed) { + mp_raise_TypeError(NULL); } } -STATIC mp_obj_t dict_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { - assert(sizeof(mp_obj_dict_it_t) <= sizeof(mp_obj_iter_buf_t)); - mp_obj_dict_it_t *o = (mp_obj_dict_it_t*)iter_buf; - o->base.type = &mp_type_polymorph_iter; - o->iternext = dict_it_iternext; - o->dict = self_in; - o->cur = 0; - return MP_OBJ_FROM_PTR(o); -} - -/******************************************************************************/ -/* dict methods */ - STATIC mp_obj_t dict_clear(mp_obj_t self_in) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); + mp_ensure_not_fixed(self); mp_map_clear(&self->map); @@ -284,6 +260,9 @@ STATIC MP_DEFINE_CONST_CLASSMETHOD_OBJ(dict_fromkeys_obj, MP_ROM_PTR(&dict_fromk STATIC mp_obj_t dict_get_helper(size_t n_args, const mp_obj_t *args, mp_map_lookup_kind_t lookup_kind) { mp_check_self(MP_OBJ_IS_DICT_TYPE(args[0])); mp_obj_dict_t *self = MP_OBJ_TO_PTR(args[0]); + if (lookup_kind != MP_MAP_LOOKUP) { + mp_ensure_not_fixed(self); + } mp_map_elem_t *elem = mp_map_lookup(&self->map, args[1], lookup_kind); mp_obj_t value; if (elem == NULL || elem->value == MP_OBJ_NULL) { @@ -326,6 +305,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(dict_setdefault_obj, 2, 3, dict_setde STATIC mp_obj_t dict_popitem(mp_obj_t self_in) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); + mp_ensure_not_fixed(self); size_t cur = 0; mp_map_elem_t *next = dict_iter_next(self, &cur); if (next == NULL) { @@ -344,6 +324,7 @@ STATIC MP_DEFINE_CONST_FUN_OBJ_1(dict_popitem_obj, dict_popitem); STATIC mp_obj_t dict_update(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs) { mp_check_self(MP_OBJ_IS_DICT_TYPE(args[0])); mp_obj_dict_t *self = MP_OBJ_TO_PTR(args[0]); + mp_ensure_not_fixed(self); mp_arg_check_num(n_args, kwargs->used, 1, 2, true); @@ -485,7 +466,7 @@ STATIC mp_obj_t dict_view_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t if (o->kind != MP_DICT_VIEW_KEYS) { return MP_OBJ_NULL; // op not supported } - if (op != MP_BINARY_OP_IN) { + if (op != MP_BINARY_OP_CONTAINS) { return MP_OBJ_NULL; // op not supported } return dict_binary_op(op, o->dict, rhs_in); @@ -527,6 +508,20 @@ STATIC mp_obj_t dict_values(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(dict_values_obj, dict_values); +/******************************************************************************/ +/* dict iterator */ + +STATIC mp_obj_t dict_getiter(mp_obj_t self_in, mp_obj_iter_buf_t *iter_buf) { + assert(sizeof(mp_obj_dict_view_it_t) <= sizeof(mp_obj_iter_buf_t)); + mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); + mp_obj_dict_view_it_t *o = (mp_obj_dict_view_it_t*)iter_buf; + o->base.type = &dict_view_it_type; + o->kind = MP_DICT_VIEW_KEYS; + o->dict = self_in; + o->cur = 0; + return MP_OBJ_FROM_PTR(o); +} + /******************************************************************************/ /* dict constructors & public C API */ @@ -595,6 +590,7 @@ size_t mp_obj_dict_len(mp_obj_t self_in) { mp_obj_t mp_obj_dict_store(mp_obj_t self_in, mp_obj_t key, mp_obj_t value) { mp_check_self(MP_OBJ_IS_DICT_TYPE(self_in)); mp_obj_dict_t *self = MP_OBJ_TO_PTR(self_in); + mp_ensure_not_fixed(self); mp_map_lookup(&self->map, key, MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = value; return self_in; } diff --git a/python/src/py/objexcept.c b/python/src/py/objexcept.c index b87609a6b..1e746bc81 100644 --- a/python/src/py/objexcept.c +++ b/python/src/py/objexcept.c @@ -43,9 +43,6 @@ // Number of traceback entries to reserve in the emergency exception buffer #define EMG_TRACEBACK_ALLOC (2 * TRACEBACK_ENTRY_LEN) -// Instance of MemoryError exception - needed by mp_malloc_fail -const mp_obj_exception_t mp_const_MemoryError_obj = {{&mp_type_MemoryError}, 0, 0, NULL, (mp_obj_tuple_t*)&mp_const_empty_tuple_obj}; - // Optionally allocated buffer for storing the first argument of an exception // allocated when the heap is locked. #if MICROPY_ENABLE_EMERGENCY_EXCEPTION_BUF @@ -96,7 +93,7 @@ mp_obj_t mp_alloc_emergency_exception_buf(mp_obj_t size_in) { // definition module-private so far, have it here. const mp_obj_exception_t mp_const_GeneratorExit_obj = {{&mp_type_GeneratorExit}, 0, 0, NULL, (mp_obj_tuple_t*)&mp_const_empty_tuple_obj}; -STATIC void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { +void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { mp_obj_exception_t *o = MP_OBJ_TO_PTR(o_in); mp_print_kind_t k = kind & ~PRINT_EXC_SUBCLASS; bool is_subclass = kind & PRINT_EXC_SUBCLASS; @@ -118,7 +115,7 @@ STATIC void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_pr if (o->base.type == &mp_type_OSError && MP_OBJ_IS_SMALL_INT(o->args->items[0])) { qstr qst = mp_errno_to_str(o->args->items[0]); if (qst != MP_QSTR_NULL) { - mp_printf(print, "[Errno %d] %q", MP_OBJ_SMALL_INT_VALUE(o->args->items[0]), qst); + mp_printf(print, "[Errno " INT_FMT "] %q", MP_OBJ_SMALL_INT_VALUE(o->args->items[0]), qst); return; } } @@ -189,7 +186,7 @@ mp_obj_t mp_obj_exception_get_value(mp_obj_t self_in) { } } -STATIC void exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { +void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { mp_obj_exception_t *self = MP_OBJ_TO_PTR(self_in); if (dest[0] != MP_OBJ_NULL) { // store/delete attribute @@ -212,37 +209,12 @@ STATIC void exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { } } -STATIC mp_obj_t exc___init__(size_t n_args, const mp_obj_t *args) { - mp_obj_exception_t *self = MP_OBJ_TO_PTR(args[0]); - mp_obj_t argst = mp_obj_new_tuple(n_args - 1, args + 1); - self->args = MP_OBJ_TO_PTR(argst); - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(exc___init___obj, 1, MP_OBJ_FUN_ARGS_MAX, exc___init__); - -STATIC const mp_rom_map_elem_t exc_locals_dict_table[] = { - { MP_ROM_QSTR(MP_QSTR___init__), MP_ROM_PTR(&exc___init___obj) }, -}; - -STATIC MP_DEFINE_CONST_DICT(exc_locals_dict, exc_locals_dict_table); - const mp_obj_type_t mp_type_BaseException = { { &mp_type_type }, .name = MP_QSTR_BaseException, .print = mp_obj_exception_print, .make_new = mp_obj_exception_make_new, - .attr = exception_attr, - .locals_dict = (mp_obj_dict_t*)&exc_locals_dict, -}; - -#define MP_DEFINE_EXCEPTION(exc_name, base_name) \ -const mp_obj_type_t mp_type_ ## exc_name = { \ - { &mp_type_type }, \ - .name = MP_QSTR_ ## exc_name, \ - .print = mp_obj_exception_print, \ - .make_new = mp_obj_exception_make_new, \ - .attr = exception_attr, \ - .parent = &mp_type_ ## base_name, \ + .attr = mp_obj_exception_attr, }; // List of all exceptions, arranged as in the table at: diff --git a/python/src/py/objexcept.h b/python/src/py/objexcept.h index f67651a7e..7c3076224 100644 --- a/python/src/py/objexcept.h +++ b/python/src/py/objexcept.h @@ -37,4 +37,17 @@ typedef struct _mp_obj_exception_t { mp_obj_tuple_t *args; } mp_obj_exception_t; +void mp_obj_exception_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind); +void mp_obj_exception_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); + +#define MP_DEFINE_EXCEPTION(exc_name, base_name) \ +const mp_obj_type_t mp_type_ ## exc_name = { \ + { &mp_type_type }, \ + .name = MP_QSTR_ ## exc_name, \ + .print = mp_obj_exception_print, \ + .make_new = mp_obj_exception_make_new, \ + .attr = mp_obj_exception_attr, \ + .parent = &mp_type_ ## base_name, \ +}; + #endif // MICROPY_INCLUDED_PY_OBJEXCEPT_H diff --git a/python/src/py/objfloat.c b/python/src/py/objfloat.c index 743287be6..e4d5a6570 100644 --- a/python/src/py/objfloat.c +++ b/python/src/py/objfloat.c @@ -137,12 +137,11 @@ STATIC mp_obj_t float_make_new(const mp_obj_type_t *type_in, size_t n_args, size return mp_obj_new_float(0); case 1: - default: - if (MP_OBJ_IS_STR(args[0])) { - // a string, parse it - size_t l; - const char *s = mp_obj_str_get_data(args[0], &l); - return mp_parse_num_decimal(s, l, false, false, NULL); + default: { + mp_buffer_info_t bufinfo; + if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { + // a textual representation, parse it + return mp_parse_num_decimal(bufinfo.buf, bufinfo.len, false, false, NULL); } else if (mp_obj_is_float(args[0])) { // a float, just return it return args[0]; @@ -150,6 +149,7 @@ STATIC mp_obj_t float_make_new(const mp_obj_type_t *type_in, size_t n_args, size // something else, try to cast it to a float return mp_obj_new_float(mp_obj_get_float(args[0])); } + } } } @@ -293,7 +293,7 @@ mp_obj_t mp_obj_float_binary_op(mp_binary_op_t op, mp_float_t lhs_val, mp_obj_t break; case MP_BINARY_OP_POWER: case MP_BINARY_OP_INPLACE_POWER: - if (lhs_val == 0 && rhs_val < 0) { + if (lhs_val == 0 && rhs_val < 0 && !isinf(rhs_val)) { goto zero_division_error; } if (lhs_val < 0 && rhs_val != MICROPY_FLOAT_C_FUN(floor)(rhs_val)) { diff --git a/python/src/py/objfun.c b/python/src/py/objfun.c index 030b3f7cb..e6d33d287 100644 --- a/python/src/py/objfun.c +++ b/python/src/py/objfun.c @@ -195,32 +195,53 @@ STATIC void dump_args(const mp_obj_t *a, size_t sz) { // than this will try to use the heap, with fallback to stack allocation. #define VM_MAX_STATE_ON_STACK (11 * sizeof(mp_uint_t)) -// Set this to enable a simple stack overflow check. +// Set this to 1 to enable a simple stack overflow check. #define VM_DETECT_STACK_OVERFLOW (0) +#define DECODE_CODESTATE_SIZE(bytecode, n_state_out_var, state_size_out_var) \ + { \ + /* bytecode prelude: state size and exception stack size */ \ + n_state_out_var = mp_decode_uint_value(bytecode); \ + size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(bytecode)); \ + \ + n_state += VM_DETECT_STACK_OVERFLOW; \ + \ + /* state size in bytes */ \ + state_size_out_var = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); \ + } + +#define INIT_CODESTATE(code_state, _fun_bc, n_args, n_kw, args) \ + code_state->fun_bc = _fun_bc; \ + code_state->ip = 0; \ + mp_setup_code_state(code_state, n_args, n_kw, args); \ + code_state->old_globals = mp_globals_get(); + #if MICROPY_STACKLESS mp_code_state_t *mp_obj_fun_bc_prepare_codestate(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_t *args) { MP_STACK_CHECK(); mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); - // bytecode prelude: state size and exception stack size - size_t n_state = mp_decode_uint_value(self->bytecode); - size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(self->bytecode)); + size_t n_state, state_size; + DECODE_CODESTATE_SIZE(self->bytecode, n_state, state_size); - // allocate state for locals and stack - size_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); mp_code_state_t *code_state; + #if MICROPY_ENABLE_PYSTACK + code_state = mp_pystack_alloc(sizeof(mp_code_state_t) + state_size); + #else + // If we use m_new_obj_var(), then on no memory, MemoryError will be + // raised. But this is not correct exception for a function call, + // RuntimeError should be raised instead. So, we use m_new_obj_var_maybe(), + // return NULL, then vm.c takes the needed action (either raise + // RuntimeError or fallback to stack allocation). code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); if (!code_state) { return NULL; } + #endif - code_state->fun_bc = self; - code_state->ip = 0; - mp_setup_code_state(code_state, n_args, n_kw, args); + INIT_CODESTATE(code_state, self, n_args, n_kw, args); // execute the byte code with the correct globals context - code_state->old_globals = mp_globals_get(); mp_globals_set(self->globals); return code_state; @@ -238,17 +259,14 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const mp_obj_fun_bc_t *self = MP_OBJ_TO_PTR(self_in); DEBUG_printf("Func n_def_args: %d\n", self->n_def_args); - // bytecode prelude: state size and exception stack size - size_t n_state = mp_decode_uint_value(self->bytecode); - size_t n_exc_stack = mp_decode_uint_value(mp_decode_uint_skip(self->bytecode)); - -#if VM_DETECT_STACK_OVERFLOW - n_state += 1; -#endif + size_t n_state, state_size; + DECODE_CODESTATE_SIZE(self->bytecode, n_state, state_size); // allocate state for locals and stack - size_t state_size = n_state * sizeof(mp_obj_t) + n_exc_stack * sizeof(mp_exc_stack_t); mp_code_state_t *code_state = NULL; + #if MICROPY_ENABLE_PYSTACK + code_state = mp_pystack_alloc(sizeof(mp_code_state_t) + state_size); + #else if (state_size > VM_MAX_STATE_ON_STACK) { code_state = m_new_obj_var_maybe(mp_code_state_t, byte, state_size); } @@ -256,13 +274,11 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const code_state = alloca(sizeof(mp_code_state_t) + state_size); state_size = 0; // indicate that we allocated using alloca } + #endif - code_state->fun_bc = self; - code_state->ip = 0; - mp_setup_code_state(code_state, n_args, n_kw, args); + INIT_CODESTATE(code_state, self, n_args, n_kw, args); // execute the byte code with the correct globals context - code_state->old_globals = mp_globals_get(); mp_globals_set(self->globals); mp_vm_return_kind_t vm_return_kind = mp_execute_bytecode(code_state, MP_OBJ_NULL); mp_globals_set(code_state->old_globals); @@ -304,10 +320,14 @@ STATIC mp_obj_t fun_bc_call(mp_obj_t self_in, size_t n_args, size_t n_kw, const result = code_state->state[n_state - 1]; } + #if MICROPY_ENABLE_PYSTACK + mp_pystack_free(code_state); + #else // free the state if it was allocated on the heap if (state_size != 0) { m_del_var(mp_code_state_t, byte, state_size, code_state); } + #endif if (vm_return_kind == MP_VM_RETURN_NORMAL) { return result; diff --git a/python/src/py/objgenerator.c b/python/src/py/objgenerator.c index bf0bbb0e6..5fd13f831 100644 --- a/python/src/py/objgenerator.c +++ b/python/src/py/objgenerator.c @@ -4,7 +4,7 @@ * The MIT License (MIT) * * Copyright (c) 2013, 2014 Damien P. George - * Copyright (c) 2014 Paul Sokolovsky + * Copyright (c) 2014-2017 Paul Sokolovsky * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -32,6 +32,7 @@ #include "py/bc.h" #include "py/objgenerator.h" #include "py/objfun.h" +#include "py/stackctrl.h" /******************************************************************************/ /* generator wrapper */ @@ -92,6 +93,7 @@ STATIC void gen_instance_print(const mp_print_t *print, mp_obj_t self_in, mp_pri } mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t throw_value, mp_obj_t *ret_val) { + MP_STACK_CHECK(); mp_check_self(MP_OBJ_IS_TYPE(self_in, &mp_type_gen_instance)); mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); if (self->code_state.ip == 0) { @@ -104,7 +106,16 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ mp_raise_TypeError("can't send non-None value to a just-started generator"); } } else { - *self->code_state.sp = send_value; + #if MICROPY_PY_GENERATOR_PEND_THROW + // If exception is pending (set using .pend_throw()), process it now. + if (*self->code_state.sp != mp_const_none) { + throw_value = *self->code_state.sp; + *self->code_state.sp = MP_OBJ_NULL; + } else + #endif + { + *self->code_state.sp = send_value; + } } mp_obj_dict_t *old_globals = mp_globals_get(); mp_globals_set(self->globals); @@ -125,9 +136,9 @@ mp_vm_return_kind_t mp_obj_gen_resume(mp_obj_t self_in, mp_obj_t send_value, mp_ case MP_VM_RETURN_YIELD: *ret_val = *self->code_state.sp; - if (*ret_val == MP_OBJ_STOP_ITERATION) { - self->code_state.ip = 0; - } + #if MICROPY_PY_GENERATOR_PEND_THROW + *self->code_state.sp = mp_const_none; + #endif break; case MP_VM_RETURN_EXCEPTION: { @@ -222,10 +233,24 @@ STATIC mp_obj_t gen_instance_close(mp_obj_t self_in) { STATIC MP_DEFINE_CONST_FUN_OBJ_1(gen_instance_close_obj, gen_instance_close); +STATIC mp_obj_t gen_instance_pend_throw(mp_obj_t self_in, mp_obj_t exc_in) { + mp_obj_gen_instance_t *self = MP_OBJ_TO_PTR(self_in); + if (self->code_state.sp == self->code_state.state - 1) { + mp_raise_TypeError("can't pend throw to just-started generator"); + } + mp_obj_t prev = *self->code_state.sp; + *self->code_state.sp = exc_in; + return prev; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(gen_instance_pend_throw_obj, gen_instance_pend_throw); + STATIC const mp_rom_map_elem_t gen_instance_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&gen_instance_close_obj) }, { MP_ROM_QSTR(MP_QSTR_send), MP_ROM_PTR(&gen_instance_send_obj) }, { MP_ROM_QSTR(MP_QSTR_throw), MP_ROM_PTR(&gen_instance_throw_obj) }, + #if MICROPY_PY_GENERATOR_PEND_THROW + { MP_ROM_QSTR(MP_QSTR_pend_throw), MP_ROM_PTR(&gen_instance_pend_throw_obj) }, + #endif }; STATIC MP_DEFINE_CONST_DICT(gen_instance_locals_dict, gen_instance_locals_dict_table); diff --git a/python/src/py/objint.c b/python/src/py/objint.c index 4f2e610a5..270e16969 100644 --- a/python/src/py/objint.c +++ b/python/src/py/objint.c @@ -222,27 +222,26 @@ size_t mp_int_format_size(size_t num_bits, int base, const char *prefix, char co char *mp_obj_int_formatted(char **buf, size_t *buf_size, size_t *fmt_size, mp_const_obj_t self_in, int base, const char *prefix, char base_char, char comma) { fmt_int_t num; + #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_NONE + // Only have small ints; get the integer value to format. + num = MP_OBJ_SMALL_INT_VALUE(self_in); + #else if (MP_OBJ_IS_SMALL_INT(self_in)) { // A small int; get the integer value to format. num = MP_OBJ_SMALL_INT_VALUE(self_in); -#if MICROPY_LONGINT_IMPL != MICROPY_LONGINT_IMPL_NONE - } else if (MP_OBJ_IS_TYPE(self_in, &mp_type_int)) { + } else { + assert(MP_OBJ_IS_TYPE(self_in, &mp_type_int)); // Not a small int. -#if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_LONGLONG + #if MICROPY_LONGINT_IMPL == MICROPY_LONGINT_IMPL_LONGLONG const mp_obj_int_t *self = self_in; // Get the value to format; mp_obj_get_int truncates to mp_int_t. num = self->val; -#else + #else // Delegate to the implementation for the long int. return mp_obj_int_formatted_impl(buf, buf_size, fmt_size, self_in, base, prefix, base_char, comma); -#endif -#endif - } else { - // Not an int. - **buf = '\0'; - *fmt_size = 0; - return *buf; + #endif } + #endif char sign = '\0'; if (num < 0) { @@ -378,7 +377,7 @@ mp_obj_t mp_obj_int_binary_op_extra_cases(mp_binary_op_t op, mp_obj_t lhs_in, mp // true acts as 0 return mp_binary_op(op, lhs_in, MP_OBJ_NEW_SMALL_INT(1)); } else if (op == MP_BINARY_OP_MULTIPLY) { - if (MP_OBJ_IS_STR(rhs_in) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_bytes) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_tuple) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_list)) { + if (MP_OBJ_IS_STR_OR_BYTES(rhs_in) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_tuple) || MP_OBJ_IS_TYPE(rhs_in, &mp_type_list)) { // multiply is commutative for these types, so delegate to them return mp_binary_op(op, rhs_in, lhs_in); } diff --git a/python/src/py/objint_longlong.c b/python/src/py/objint_longlong.c index 2e567c572..cb8d1672d 100644 --- a/python/src/py/objint_longlong.c +++ b/python/src/py/objint_longlong.c @@ -124,10 +124,9 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i if (MP_OBJ_IS_SMALL_INT(lhs_in)) { lhs_val = MP_OBJ_SMALL_INT_VALUE(lhs_in); - } else if (MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)) { - lhs_val = ((mp_obj_int_t*)lhs_in)->val; } else { - return MP_OBJ_NULL; // op not supported + assert(MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)); + lhs_val = ((mp_obj_int_t*)lhs_in)->val; } if (MP_OBJ_IS_SMALL_INT(rhs_in)) { @@ -151,9 +150,15 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return mp_obj_new_int_from_ll(lhs_val * rhs_val); case MP_BINARY_OP_FLOOR_DIVIDE: case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: + if (rhs_val == 0) { + goto zero_division; + } return mp_obj_new_int_from_ll(lhs_val / rhs_val); case MP_BINARY_OP_MODULO: case MP_BINARY_OP_INPLACE_MODULO: + if (rhs_val == 0) { + goto zero_division; + } return mp_obj_new_int_from_ll(lhs_val % rhs_val); case MP_BINARY_OP_AND: @@ -210,6 +215,9 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i default: return MP_OBJ_NULL; // op not supported } + +zero_division: + mp_raise_msg(&mp_type_ZeroDivisionError, "division by zero"); } mp_obj_t mp_obj_new_int(mp_int_t value) { diff --git a/python/src/py/objint_mpz.c b/python/src/py/objint_mpz.c index 7b5cb0b9d..0f05c84f4 100644 --- a/python/src/py/objint_mpz.c +++ b/python/src/py/objint_mpz.c @@ -170,11 +170,9 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i if (MP_OBJ_IS_SMALL_INT(lhs_in)) { mpz_init_fixed_from_int(&z_int, z_int_dig, MPZ_NUM_DIG_FOR_INT, MP_OBJ_SMALL_INT_VALUE(lhs_in)); zlhs = &z_int; - } else if (MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)) { - zlhs = &((mp_obj_int_t*)MP_OBJ_TO_PTR(lhs_in))->mpz; } else { - // unsupported type - return MP_OBJ_NULL; + assert(MP_OBJ_IS_TYPE(lhs_in, &mp_type_int)); + zlhs = &((mp_obj_int_t*)MP_OBJ_TO_PTR(lhs_in))->mpz; } // if rhs is small int, then lhs was not (otherwise mp_binary_op handles it) @@ -207,7 +205,7 @@ mp_obj_t mp_obj_int_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return mp_obj_new_float(flhs / frhs); #endif - } else if (op >= MP_BINARY_OP_INPLACE_OR) { + } else if (op >= MP_BINARY_OP_INPLACE_OR && op < MP_BINARY_OP_CONTAINS) { mp_obj_int_t *res = mp_obj_int_new_mpz(); switch (op) { diff --git a/python/src/py/objlist.h b/python/src/py/objlist.h index 28b5495a9..a43663db7 100644 --- a/python/src/py/objlist.h +++ b/python/src/py/objlist.h @@ -35,4 +35,6 @@ typedef struct _mp_obj_list_t { mp_obj_t *items; } mp_obj_list_t; +void mp_obj_list_init(mp_obj_list_t *o, size_t n); + #endif // MICROPY_INCLUDED_PY_OBJLIST_H diff --git a/python/src/py/objmodule.c b/python/src/py/objmodule.c index f9363e379..c4aba3a7b 100644 --- a/python/src/py/objmodule.c +++ b/python/src/py/objmodule.c @@ -247,17 +247,7 @@ mp_obj_t mp_module_get(qstr module_name) { if (el == NULL) { return MP_OBJ_NULL; } - - if (MICROPY_MODULE_BUILTIN_INIT) { - // look for __init__ and call it if it exists - mp_obj_t dest[2]; - mp_load_method_maybe(el->value, MP_QSTR___init__, dest); - if (dest[0] != MP_OBJ_NULL) { - mp_call_method_n_kw(0, 0, dest); - // register module so __init__ is not called again - mp_module_register(module_name, el->value); - } - } + mp_module_call_init(module_name, el->value); } // module found, return it @@ -268,3 +258,19 @@ void mp_module_register(qstr qst, mp_obj_t module) { mp_map_t *mp_loaded_modules_map = &MP_STATE_VM(mp_loaded_modules_dict).map; mp_map_lookup(mp_loaded_modules_map, MP_OBJ_NEW_QSTR(qst), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND)->value = module; } + +#if MICROPY_MODULE_BUILTIN_INIT +void mp_module_call_init(qstr module_name, mp_obj_t module_obj) { + // Look for __init__ and call it if it exists + mp_obj_t dest[2]; + mp_load_method_maybe(module_obj, MP_QSTR___init__, dest); + if (dest[0] != MP_OBJ_NULL) { + mp_call_method_n_kw(0, 0, dest); + // Register module so __init__ is not called again. + // If a module can be referenced by more than one name (eg due to weak links) + // then __init__ will still be called for each distinct import, and it's then + // up to the particular module to make sure it's __init__ code only runs once. + mp_module_register(module_name, module_obj); + } +} +#endif diff --git a/python/src/py/objmodule.h b/python/src/py/objmodule.h index b5c07dc33..b7702ec50 100644 --- a/python/src/py/objmodule.h +++ b/python/src/py/objmodule.h @@ -34,4 +34,13 @@ extern const mp_map_t mp_builtin_module_weak_links_map; mp_obj_t mp_module_get(qstr module_name); void mp_module_register(qstr qstr, mp_obj_t module); +#if MICROPY_MODULE_BUILTIN_INIT +void mp_module_call_init(qstr module_name, mp_obj_t module_obj); +#else +static inline void mp_module_call_init(qstr module_name, mp_obj_t module_obj) { + (void)module_name; + (void)module_obj; +} +#endif + #endif // MICROPY_INCLUDED_PY_OBJMODULE_H diff --git a/python/src/py/objnamedtuple.c b/python/src/py/objnamedtuple.c index 38daccdf2..e7de899cf 100644 --- a/python/src/py/objnamedtuple.c +++ b/python/src/py/objnamedtuple.c @@ -30,20 +30,11 @@ #include "py/objtuple.h" #include "py/runtime.h" #include "py/objstr.h" +#include "py/objnamedtuple.h" #if MICROPY_PY_COLLECTIONS -typedef struct _mp_obj_namedtuple_type_t { - mp_obj_type_t base; - size_t n_fields; - qstr fields[]; -} mp_obj_namedtuple_type_t; - -typedef struct _mp_obj_namedtuple_t { - mp_obj_tuple_t tuple; -} mp_obj_namedtuple_t; - -STATIC size_t namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr name) { +size_t mp_obj_namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr name) { for (size_t i = 0; i < type->n_fields; i++) { if (type->fields[i] == name) { return i; @@ -52,6 +43,23 @@ STATIC size_t namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr n return (size_t)-1; } +#if MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT +STATIC mp_obj_t namedtuple_asdict(mp_obj_t self_in) { + mp_obj_namedtuple_t *self = MP_OBJ_TO_PTR(self_in); + const qstr *fields = ((mp_obj_namedtuple_type_t*)self->tuple.base.type)->fields; + mp_obj_t dict = mp_obj_new_dict(self->tuple.len); + //make it an OrderedDict + mp_obj_dict_t *dictObj = MP_OBJ_TO_PTR(dict); + dictObj->base.type = &mp_type_ordereddict; + dictObj->map.is_ordered = 1; + for (size_t i = 0; i < self->tuple.len; ++i) { + mp_obj_dict_store(dict, MP_OBJ_NEW_QSTR(fields[i]), self->tuple.items[i]); + } + return dict; +} +MP_DEFINE_CONST_FUN_OBJ_1(namedtuple_asdict_obj, namedtuple_asdict); +#endif + STATIC void namedtuple_print(const mp_print_t *print, mp_obj_t o_in, mp_print_kind_t kind) { (void)kind; mp_obj_namedtuple_t *o = MP_OBJ_TO_PTR(o_in); @@ -64,7 +72,14 @@ STATIC void namedtuple_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (dest[0] == MP_OBJ_NULL) { // load attribute mp_obj_namedtuple_t *self = MP_OBJ_TO_PTR(self_in); - size_t id = namedtuple_find_field((mp_obj_namedtuple_type_t*)self->tuple.base.type, attr); + #if MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT + if (attr == MP_QSTR__asdict) { + dest[0] = MP_OBJ_FROM_PTR(&namedtuple_asdict_obj); + dest[1] = self_in; + return; + } + #endif + size_t id = mp_obj_namedtuple_find_field((mp_obj_namedtuple_type_t*)self->tuple.base.type, attr); if (id == (size_t)-1) { return; } @@ -104,7 +119,7 @@ STATIC mp_obj_t namedtuple_make_new(const mp_obj_type_t *type_in, size_t n_args, memset(&tuple->items[n_args], 0, sizeof(mp_obj_t) * n_kw); for (size_t i = n_args; i < n_args + 2 * n_kw; i += 2) { qstr kw = mp_obj_str_get_qstr(args[i]); - size_t id = namedtuple_find_field(type, kw); + size_t id = mp_obj_namedtuple_find_field(type, kw); if (id == (size_t)-1) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_arg_error_terse_mismatch(); @@ -127,9 +142,18 @@ STATIC mp_obj_t namedtuple_make_new(const mp_obj_type_t *type_in, size_t n_args, return MP_OBJ_FROM_PTR(tuple); } -STATIC mp_obj_t mp_obj_new_namedtuple_type(qstr name, size_t n_fields, mp_obj_t *fields) { +mp_obj_namedtuple_type_t *mp_obj_new_namedtuple_base(size_t n_fields, mp_obj_t *fields) { mp_obj_namedtuple_type_t *o = m_new_obj_var(mp_obj_namedtuple_type_t, qstr, n_fields); memset(&o->base, 0, sizeof(o->base)); + o->n_fields = n_fields; + for (size_t i = 0; i < n_fields; i++) { + o->fields[i] = mp_obj_str_get_qstr(fields[i]); + } + return o; +} + +STATIC mp_obj_t mp_obj_new_namedtuple_type(qstr name, size_t n_fields, mp_obj_t *fields) { + mp_obj_namedtuple_type_t *o = mp_obj_new_namedtuple_base(n_fields, fields); o->base.base.type = &mp_type_type; o->base.name = name; o->base.print = namedtuple_print; @@ -140,10 +164,6 @@ STATIC mp_obj_t mp_obj_new_namedtuple_type(qstr name, size_t n_fields, mp_obj_t o->base.subscr = mp_obj_tuple_subscr; o->base.getiter = mp_obj_tuple_getiter; o->base.parent = &mp_type_tuple; - o->n_fields = n_fields; - for (size_t i = 0; i < n_fields; i++) { - o->fields[i] = mp_obj_str_get_qstr(fields[i]); - } return MP_OBJ_FROM_PTR(o); } diff --git a/python/src/py/objnamedtuple.h b/python/src/py/objnamedtuple.h new file mode 100644 index 000000000..d32af35af --- /dev/null +++ b/python/src/py/objnamedtuple.h @@ -0,0 +1,44 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2014-2017 Paul Sokolovsky + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H +#define MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H + +#include "py/objtuple.h" + +typedef struct _mp_obj_namedtuple_type_t { + mp_obj_type_t base; + size_t n_fields; + qstr fields[]; +} mp_obj_namedtuple_type_t; + +typedef struct _mp_obj_namedtuple_t { + mp_obj_tuple_t tuple; +} mp_obj_namedtuple_t; + +size_t mp_obj_namedtuple_find_field(const mp_obj_namedtuple_type_t *type, qstr name); +mp_obj_namedtuple_type_t *mp_obj_new_namedtuple_base(size_t n_fields, mp_obj_t *fields); + +#endif // MICROPY_INCLUDED_PY_OBJNAMEDTUPLE_H diff --git a/python/src/py/objobject.c b/python/src/py/objobject.c index 49d2ec62e..265fcfbf2 100644 --- a/python/src/py/objobject.c +++ b/python/src/py/objobject.c @@ -52,9 +52,12 @@ STATIC mp_obj_t object___new__(mp_obj_t cls) { if (!MP_OBJ_IS_TYPE(cls, &mp_type_type) || !mp_obj_is_instance_type((mp_obj_type_t*)MP_OBJ_TO_PTR(cls))) { mp_raise_TypeError("__new__ arg must be a user-type"); } - mp_obj_t o = MP_OBJ_SENTINEL; - mp_obj_t res = mp_obj_instance_make_new(MP_OBJ_TO_PTR(cls), 1, 0, &o); - return res; + // This executes only "__new__" part of instance creation. + // TODO: This won't work well for classes with native bases. + // TODO: This is a hack, should be resolved along the lines of + // https://github.com/micropython/micropython/issues/606#issuecomment-43685883 + const mp_obj_type_t *native_base; + return MP_OBJ_FROM_PTR(mp_obj_new_instance(MP_OBJ_TO_PTR(cls), &native_base)); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(object___new___fun_obj, object___new__); STATIC MP_DEFINE_CONST_STATICMETHOD_OBJ(object___new___obj, MP_ROM_PTR(&object___new___fun_obj)); diff --git a/python/src/py/objrange.c b/python/src/py/objrange.c index 3874adb11..86aa0ccfe 100644 --- a/python/src/py/objrange.c +++ b/python/src/py/objrange.c @@ -138,6 +138,24 @@ STATIC mp_obj_t range_unary_op(mp_unary_op_t op, mp_obj_t self_in) { } } +#if MICROPY_PY_BUILTINS_RANGE_BINOP +STATIC mp_obj_t range_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in) { + if (!MP_OBJ_IS_TYPE(rhs_in, &mp_type_range) || op != MP_BINARY_OP_EQUAL) { + return MP_OBJ_NULL; // op not supported + } + mp_obj_range_t *lhs = MP_OBJ_TO_PTR(lhs_in); + mp_obj_range_t *rhs = MP_OBJ_TO_PTR(rhs_in); + mp_int_t lhs_len = range_len(lhs); + mp_int_t rhs_len = range_len(rhs); + return mp_obj_new_bool( + lhs_len == rhs_len + && (lhs_len == 0 + || (lhs->start == rhs->start + && (lhs_len == 1 || lhs->step == rhs->step))) + ); +} +#endif + STATIC mp_obj_t range_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { if (value == MP_OBJ_SENTINEL) { // load @@ -195,6 +213,9 @@ const mp_obj_type_t mp_type_range = { .print = range_print, .make_new = range_make_new, .unary_op = range_unary_op, + #if MICROPY_PY_BUILTINS_RANGE_BINOP + .binary_op = range_binary_op, + #endif .subscr = range_subscr, .getiter = range_getiter, #if MICROPY_PY_BUILTINS_RANGE_ATTRS diff --git a/python/src/py/objset.c b/python/src/py/objset.c index 6ed15c791..799ba9df0 100644 --- a/python/src/py/objset.c +++ b/python/src/py/objset.c @@ -351,11 +351,9 @@ STATIC mp_obj_t set_issuperset_proper(mp_obj_t self_in, mp_obj_t other_in) { } STATIC mp_obj_t set_equal(mp_obj_t self_in, mp_obj_t other_in) { + assert(is_set_or_frozenset(other_in)); check_set_or_frozenset(self_in); mp_obj_set_t *self = MP_OBJ_TO_PTR(self_in); - if (!is_set_or_frozenset(other_in)) { - return mp_const_false; - } mp_obj_set_t *other = MP_OBJ_TO_PTR(other_in); if (self->set.used != other->set.used) { return mp_const_false; @@ -461,7 +459,7 @@ STATIC mp_obj_t set_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { #else bool update = true; #endif - if (op != MP_BINARY_OP_IN && !is_set_or_frozenset(rhs)) { + if (op != MP_BINARY_OP_CONTAINS && !is_set_or_frozenset(rhs)) { // For all ops except containment the RHS must be a set/frozenset return MP_OBJ_NULL; } @@ -507,7 +505,7 @@ STATIC mp_obj_t set_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { return set_issubset(lhs, rhs); case MP_BINARY_OP_MORE_EQUAL: return set_issuperset(lhs, rhs); - case MP_BINARY_OP_IN: { + case MP_BINARY_OP_CONTAINS: { mp_obj_set_t *o = MP_OBJ_TO_PTR(lhs); mp_obj_t elem = mp_set_lookup(&o->set, rhs, MP_MAP_LOOKUP); return mp_obj_new_bool(elem != MP_OBJ_NULL); diff --git a/python/src/py/objstr.c b/python/src/py/objstr.c index 51da7a418..da925234e 100644 --- a/python/src/py/objstr.c +++ b/python/src/py/objstr.c @@ -164,7 +164,14 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif - mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_of_type(type, NULL, str_len)); + + // Check if a qstr with this data already exists + qstr q = qstr_find_strn((const char*)str_data, str_len); + if (q != MP_QSTR_NULL) { + return MP_OBJ_NEW_QSTR(q); + } + + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(type, NULL, str_len)); o->data = str_data; o->hash = str_hash; return MP_OBJ_FROM_PTR(o); @@ -176,7 +183,7 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_ mp_raise_msg(&mp_type_UnicodeError, NULL); } #endif - return mp_obj_new_str(bufinfo.buf, bufinfo.len, false); + return mp_obj_new_str(bufinfo.buf, bufinfo.len); } } } @@ -205,7 +212,7 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size if (str_hash == 0) { str_hash = qstr_compute_hash(str_data, str_len); } - mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_of_type(&mp_type_bytes, NULL, str_len)); + mp_obj_str_t *o = MP_OBJ_TO_PTR(mp_obj_new_str_copy(&mp_type_bytes, NULL, str_len)); o->data = str_data; o->hash = str_hash; return MP_OBJ_FROM_PTR(o); @@ -216,7 +223,10 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size } if (MP_OBJ_IS_SMALL_INT(args[0])) { - uint len = MP_OBJ_SMALL_INT_VALUE(args[0]); + mp_int_t len = MP_OBJ_SMALL_INT_VALUE(args[0]); + if (len < 0) { + mp_raise_ValueError(NULL); + } vstr_t vstr; vstr_init_len(&vstr, len); memset(vstr.buf, 0, len); @@ -226,7 +236,7 @@ STATIC mp_obj_t bytes_make_new(const mp_obj_type_t *type_in, size_t n_args, size // check if argument has the buffer protocol mp_buffer_info_t bufinfo; if (mp_get_buffer(args[0], &bufinfo, MP_BUFFER_READ)) { - return mp_obj_new_str_of_type(&mp_type_bytes, bufinfo.buf, bufinfo.len); + return mp_obj_new_bytes(bufinfo.buf, bufinfo.len); } vstr_t vstr; @@ -377,8 +387,7 @@ mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_i return mp_obj_new_str_from_vstr(lhs_type, &vstr); } - case MP_BINARY_OP_IN: - /* NOTE `a in b` is `b.__contains__(a)` */ + case MP_BINARY_OP_CONTAINS: return mp_obj_new_bool(find_subbytes(lhs_data, lhs_len, rhs_data, rhs_len, 1) != NULL); //case MP_BINARY_OP_NOT_EQUAL: // This is never passed here @@ -423,7 +432,7 @@ STATIC mp_obj_t bytes_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { if (MICROPY_PY_BUILTINS_STR_UNICODE || type == &mp_type_bytes) { return MP_OBJ_NEW_SMALL_INT(self_data[index_val]); } else { - return mp_obj_new_str((char*)&self_data[index_val], 1, true); + return mp_obj_new_str_via_qstr((char*)&self_data[index_val], 1); } } else { return MP_OBJ_NULL; // op not supported @@ -654,9 +663,7 @@ STATIC mp_obj_t str_rsplit(size_t n_args, const mp_obj_t *args) { } res->items[idx--] = mp_obj_new_str_of_type(self_type, s + sep_len, last - s - sep_len); last = s; - if (splits > 0) { - splits--; - } + splits--; } if (idx != 0) { // We split less parts than split limit, now go cleanup surplus @@ -692,8 +699,13 @@ STATIC mp_obj_t str_finder(size_t n_args, const mp_obj_t *args, int direction, b end = str_index_to_ptr(self_type, haystack, haystack_len, args[3], true); } + if (end < start) { + goto out_error; + } + const byte *p = find_subbytes(start, end - start, needle, needle_len, direction); if (p == NULL) { + out_error: // not found if (is_index) { mp_raise_ValueError("substring not found"); @@ -1046,7 +1058,7 @@ STATIC vstr_t mp_obj_str_format_helper(const char *str, const char *top, int *ar } else { const char *lookup; for (lookup = field_name; lookup < field_name_top && *lookup != '.' && *lookup != '['; lookup++); - mp_obj_t field_q = mp_obj_new_str(field_name, lookup - field_name, true/*?*/); + mp_obj_t field_q = mp_obj_new_str_via_qstr(field_name, lookup - field_name); // should it be via qstr? field_name = lookup; mp_map_elem_t *key_elem = mp_map_lookup(kwargs, field_q, MP_MAP_LOOKUP); if (key_elem == NULL) { @@ -1413,7 +1425,7 @@ STATIC mp_obj_t str_modulo_format(mp_obj_t pattern, size_t n_args, const mp_obj_ } ++str; } - mp_obj_t k_obj = mp_obj_new_str((const char*)key, str - key, true); + mp_obj_t k_obj = mp_obj_new_str_via_qstr((const char*)key, str - key); arg = mp_obj_dict_get(dict, k_obj); str++; } @@ -1698,7 +1710,7 @@ STATIC mp_obj_t str_count(size_t n_args, const mp_obj_t *args) { // if needle_len is zero then we count each gap between characters as an occurrence if (needle_len == 0) { - return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char*)start, end - start) + 1); + return MP_OBJ_NEW_SMALL_INT(utf8_charlen(start, end - start) + 1); } // count the occurrences @@ -1977,8 +1989,9 @@ const mp_obj_type_t mp_type_bytes = { const mp_obj_str_t mp_const_empty_bytes_obj = {{&mp_type_bytes}, 0, 0, (const byte*)""}; // Create a str/bytes object using the given data. New memory is allocated and -// the data is copied across. -mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, size_t len) { +// the data is copied across. This function should only be used if the type is bytes, +// or if the type is str and the string data is known to be not interned. +mp_obj_t mp_obj_new_str_copy(const mp_obj_type_t *type, const byte* data, size_t len) { mp_obj_str_t *o = m_new_obj(mp_obj_str_t); o->base.type = type; o->len = len; @@ -1992,6 +2005,22 @@ mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, siz return MP_OBJ_FROM_PTR(o); } +// Create a str/bytes object using the given data. If the type is str and the string +// data is already interned, then a qstr object is returned. Otherwise new memory is +// allocated for the object and the data is copied across. +mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, size_t len) { + if (type == &mp_type_str) { + return mp_obj_new_str((const char*)data, len); + } else { + return mp_obj_new_bytes(data, len); + } +} + +// Create a str using a qstr to store the data; may use existing or new qstr. +mp_obj_t mp_obj_new_str_via_qstr(const char* data, size_t len) { + return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); +} + // Create a str/bytes object from the given vstr. The vstr buffer is resized to // the exact length required and then reused for the str/bytes object. The vstr // is cleared and can safely be passed to vstr_free if it was heap allocated. @@ -2022,29 +2051,30 @@ mp_obj_t mp_obj_new_str_from_vstr(const mp_obj_type_t *type, vstr_t *vstr) { return MP_OBJ_FROM_PTR(o); } -mp_obj_t mp_obj_new_str(const char* data, size_t len, bool make_qstr_if_not_already) { - if (make_qstr_if_not_already) { - // use existing, or make a new qstr - return MP_OBJ_NEW_QSTR(qstr_from_strn(data, len)); +mp_obj_t mp_obj_new_str(const char* data, size_t len) { + qstr q = qstr_find_strn(data, len); + if (q != MP_QSTR_NULL) { + // qstr with this data already exists + return MP_OBJ_NEW_QSTR(q); } else { - qstr q = qstr_find_strn(data, len); - if (q != MP_QSTR_NULL) { - // qstr with this data already exists - return MP_OBJ_NEW_QSTR(q); - } else { - // no existing qstr, don't make one - return mp_obj_new_str_of_type(&mp_type_str, (const byte*)data, len); - } + // no existing qstr, don't make one + return mp_obj_new_str_copy(&mp_type_str, (const byte*)data, len); } } mp_obj_t mp_obj_str_intern(mp_obj_t str) { GET_STR_DATA_LEN(str, data, len); - return MP_OBJ_NEW_QSTR(qstr_from_strn((const char*)data, len)); + return mp_obj_new_str_via_qstr((const char*)data, len); +} + +mp_obj_t mp_obj_str_intern_checked(mp_obj_t obj) { + size_t len; + const char *data = mp_obj_str_get_data(obj, &len); + return mp_obj_new_str_via_qstr((const char*)data, len); } mp_obj_t mp_obj_new_bytes(const byte* data, size_t len) { - return mp_obj_new_str_of_type(&mp_type_bytes, data, len); + return mp_obj_new_str_copy(&mp_type_bytes, data, len); } bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { @@ -2066,7 +2096,7 @@ bool mp_obj_str_equal(mp_obj_t s1, mp_obj_t s2) { } } -STATIC void bad_implicit_conversion(mp_obj_t self_in) { +STATIC NORETURN void bad_implicit_conversion(mp_obj_t self_in) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_raise_TypeError("can't convert to str implicitly"); } else { @@ -2138,7 +2168,7 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { mp_obj_str8_it_t *self = MP_OBJ_TO_PTR(self_in); GET_STR_DATA_LEN(self->str, str, len); if (self->cur < len) { - mp_obj_t o_out = mp_obj_new_str((const char*)str + self->cur, 1, true); + mp_obj_t o_out = mp_obj_new_str_via_qstr((const char*)str + self->cur, 1); self->cur += 1; return o_out; } else { diff --git a/python/src/py/objstr.h b/python/src/py/objstr.h index 82501a763..4e55cad09 100644 --- a/python/src/py/objstr.h +++ b/python/src/py/objstr.h @@ -65,6 +65,7 @@ mp_obj_t mp_obj_str_make_new(const mp_obj_type_t *type_in, size_t n_args, size_t void mp_str_print_json(const mp_print_t *print, const byte *str_data, size_t str_len); mp_obj_t mp_obj_str_format(size_t n_args, const mp_obj_t *args, mp_map_t *kwargs); mp_obj_t mp_obj_str_split(size_t n_args, const mp_obj_t *args); +mp_obj_t mp_obj_new_str_copy(const mp_obj_type_t *type, const byte* data, size_t len); mp_obj_t mp_obj_new_str_of_type(const mp_obj_type_t *type, const byte* data, size_t len); mp_obj_t mp_obj_str_binary_op(mp_binary_op_t op, mp_obj_t lhs_in, mp_obj_t rhs_in); diff --git a/python/src/py/objstringio.c b/python/src/py/objstringio.c index 5c50aa317..b405ee21e 100644 --- a/python/src/py/objstringio.c +++ b/python/src/py/objstringio.c @@ -143,6 +143,17 @@ STATIC mp_uint_t stringio_ioctl(mp_obj_t o_in, mp_uint_t request, uintptr_t arg, } case MP_STREAM_FLUSH: return 0; + case MP_STREAM_CLOSE: + #if MICROPY_CPYTHON_COMPAT + vstr_free(o->vstr); + o->vstr = NULL; + #else + vstr_clear(o->vstr); + o->vstr->alloc = 0; + o->vstr->len = 0; + o->pos = 0; + #endif + return 0; default: *errcode = MP_EINVAL; return MP_STREAM_ERROR; @@ -159,24 +170,9 @@ STATIC mp_obj_t stringio_getvalue(mp_obj_t self_in) { } STATIC MP_DEFINE_CONST_FUN_OBJ_1(stringio_getvalue_obj, stringio_getvalue); -STATIC mp_obj_t stringio_close(mp_obj_t self_in) { - mp_obj_stringio_t *self = MP_OBJ_TO_PTR(self_in); -#if MICROPY_CPYTHON_COMPAT - vstr_free(self->vstr); - self->vstr = NULL; -#else - vstr_clear(self->vstr); - self->vstr->alloc = 0; - self->vstr->len = 0; - self->pos = 0; -#endif - return mp_const_none; -} -STATIC MP_DEFINE_CONST_FUN_OBJ_1(stringio_close_obj, stringio_close); - STATIC mp_obj_t stringio___exit__(size_t n_args, const mp_obj_t *args) { (void)n_args; - return stringio_close(args[0]); + return mp_stream_close(args[0]); } STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(stringio___exit___obj, 4, 4, stringio___exit__); @@ -233,7 +229,7 @@ STATIC const mp_rom_map_elem_t stringio_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_write), MP_ROM_PTR(&mp_stream_write_obj) }, { MP_ROM_QSTR(MP_QSTR_seek), MP_ROM_PTR(&mp_stream_seek_obj) }, { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&mp_stream_flush_obj) }, - { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&stringio_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&mp_stream_close_obj) }, { MP_ROM_QSTR(MP_QSTR_getvalue), MP_ROM_PTR(&stringio_getvalue_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&stringio___exit___obj) }, diff --git a/python/src/py/objstrunicode.c b/python/src/py/objstrunicode.c index 29f7695b7..badb569d7 100644 --- a/python/src/py/objstrunicode.c +++ b/python/src/py/objstrunicode.c @@ -104,7 +104,7 @@ STATIC mp_obj_t uni_unary_op(mp_unary_op_t op, mp_obj_t self_in) { case MP_UNARY_OP_BOOL: return mp_obj_new_bool(str_len != 0); case MP_UNARY_OP_LEN: - return MP_OBJ_NEW_SMALL_INT(unichar_charlen((const char *)str_data, str_len)); + return MP_OBJ_NEW_SMALL_INT(utf8_charlen(str_data, str_len)); default: return MP_OBJ_NULL; // op not supported } @@ -216,7 +216,7 @@ STATIC mp_obj_t str_subscr(mp_obj_t self_in, mp_obj_t index, mp_obj_t value) { ++len; } } - return mp_obj_new_str((const char*)s, len, true); // This will create a one-character string + return mp_obj_new_str_via_qstr((const char*)s, len); // This will create a one-character string } else { return MP_OBJ_NULL; // op not supported } @@ -291,7 +291,7 @@ STATIC mp_obj_t str_it_iternext(mp_obj_t self_in) { if (self->cur < len) { const byte *cur = str + self->cur; const byte *end = utf8_next_char(str + self->cur); - mp_obj_t o_out = mp_obj_new_str((const char*)cur, end - cur, true); + mp_obj_t o_out = mp_obj_new_str_via_qstr((const char*)cur, end - cur); self->cur += end - cur; return o_out; } else { diff --git a/python/src/py/objtype.c b/python/src/py/objtype.c index 6e2ab6c9a..7df349ce9 100644 --- a/python/src/py/objtype.c +++ b/python/src/py/objtype.c @@ -46,14 +46,6 @@ STATIC mp_obj_t static_class_method_make_new(const mp_obj_type_t *self_in, size_ /******************************************************************************/ // instance object -STATIC mp_obj_t mp_obj_new_instance(const mp_obj_type_t *class, size_t subobjs) { - mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, subobjs); - o->base.type = class; - mp_map_init(&o->members, 0); - mp_seq_clear(o->subobj, 0, subobjs, sizeof(*o->subobj)); - return MP_OBJ_FROM_PTR(o); -} - STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_type_t **last_native_base) { int count = 0; for (;;) { @@ -67,6 +59,7 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t } else if (type->parent == NULL) { // No parents so end search here. return count; + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)type->parent)->type == &mp_type_tuple) { // Multiple parents, search through them all recursively. const mp_obj_tuple_t *parent_tuple = type->parent; @@ -78,6 +71,7 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t count += instance_count_native_bases(bt, last_native_base); } return count; + #endif } else { // A single parent, use iteration to continue the search. type = type->parent; @@ -85,6 +79,35 @@ STATIC int instance_count_native_bases(const mp_obj_type_t *type, const mp_obj_t } } +// This wrapper function is allows a subclass of a native type to call the +// __init__() method (corresponding to type->make_new) of the native type. +STATIC mp_obj_t native_base_init_wrapper(size_t n_args, const mp_obj_t *args) { + mp_obj_instance_t *self = MP_OBJ_TO_PTR(args[0]); + const mp_obj_type_t *native_base = NULL; + instance_count_native_bases(self->base.type, &native_base); + self->subobj[0] = native_base->make_new(native_base, n_args - 1, 0, args + 1); + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(native_base_init_wrapper_obj, 1, MP_OBJ_FUN_ARGS_MAX, native_base_init_wrapper); + +#if !MICROPY_CPYTHON_COMPAT +STATIC +#endif +mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *class, const mp_obj_type_t **native_base) { + size_t num_native_bases = instance_count_native_bases(class, native_base); + assert(num_native_bases < 2); + mp_obj_instance_t *o = m_new_obj_var(mp_obj_instance_t, mp_obj_t, num_native_bases); + o->base.type = class; + mp_map_init(&o->members, 0); + // Initialise the native base-class slot (should be 1 at most) with a valid + // object. It doesn't matter which object, so long as it can be uniquely + // distinguished from a native class that is initialised. + if (num_native_bases != 0) { + o->subobj[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); + } + return o; +} + // TODO // This implements depth-first left-to-right MRO, which is not compliant with Python3 MRO // http://python-history.blogspot.com/2010/06/method-resolution-order.html @@ -172,6 +195,7 @@ STATIC void mp_obj_class_lookup(struct class_lookup_data *lookup, const mp_obj_ if (type->parent == NULL) { DEBUG_printf("mp_obj_class_lookup: No more parents\n"); return; + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)type->parent)->type == &mp_type_tuple) { const mp_obj_tuple_t *parent_tuple = type->parent; const mp_obj_t *item = parent_tuple->items; @@ -192,6 +216,7 @@ STATIC void mp_obj_class_lookup(struct class_lookup_data *lookup, const mp_obj_ // search last base (simple tail recursion elimination) assert(MP_OBJ_IS_TYPE(*item, &mp_type_type)); type = (mp_obj_type_t*)MP_OBJ_TO_PTR(*item); + #endif } else { type = type->parent; } @@ -247,20 +272,6 @@ STATIC void instance_print(const mp_print_t *print, mp_obj_t self_in, mp_print_k mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size_t n_kw, const mp_obj_t *args) { assert(mp_obj_is_instance_type(self)); - const mp_obj_type_t *native_base; - size_t num_native_bases = instance_count_native_bases(self, &native_base); - assert(num_native_bases < 2); - - mp_obj_instance_t *o = MP_OBJ_TO_PTR(mp_obj_new_instance(self, num_native_bases)); - - // This executes only "__new__" part of instance creation. - // TODO: This won't work well for classes with native bases. - // TODO: This is a hack, should be resolved along the lines of - // https://github.com/micropython/micropython/issues/606#issuecomment-43685883 - if (n_args == 1 && *args == MP_OBJ_SENTINEL) { - return MP_OBJ_FROM_PTR(o); - } - // look for __new__ function mp_obj_t init_fn[2] = {MP_OBJ_NULL}; struct class_lookup_data lookup = { @@ -272,13 +283,22 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size }; mp_obj_class_lookup(&lookup, self); - mp_obj_t new_ret = MP_OBJ_FROM_PTR(o); - if (init_fn[0] == MP_OBJ_SENTINEL) { - // Native type's constructor is what wins - it gets all our arguments, - // and none Python classes are initialized at all. - o->subobj[0] = native_base->make_new(native_base, n_args, n_kw, args); - } else if (init_fn[0] != MP_OBJ_NULL) { - // now call Python class __new__ function with all args + const mp_obj_type_t *native_base = NULL; + mp_obj_instance_t *o; + if (init_fn[0] == MP_OBJ_NULL || init_fn[0] == MP_OBJ_SENTINEL) { + // Either there is no __new__() method defined or there is a native + // constructor. In both cases create a blank instance. + o = mp_obj_new_instance(self, &native_base); + + // Since type->make_new() implements both __new__() and __init__() in + // one go, of which the latter may be overridden by the Python subclass, + // we defer (see the end of this function) the call of the native + // constructor to give a chance for the Python __init__() method to call + // said native constructor. + + } else { + // Call Python class __new__ function with all args to create an instance + mp_obj_t new_ret; if (n_args == 0 && n_kw == 0) { mp_obj_t args2[1] = {MP_OBJ_FROM_PTR(self)}; new_ret = mp_call_function_n_kw(init_fn[0], 1, 0, args2); @@ -290,17 +310,20 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size m_del(mp_obj_t, args2, 1 + n_args + 2 * n_kw); } - } + // https://docs.python.org/3.4/reference/datamodel.html#object.__new__ + // "If __new__() does not return an instance of cls, then the new + // instance's __init__() method will not be invoked." + if (mp_obj_get_type(new_ret) != self) { + return new_ret; + } - // https://docs.python.org/3.4/reference/datamodel.html#object.__new__ - // "If __new__() does not return an instance of cls, then the new instance's __init__() method will not be invoked." - if (mp_obj_get_type(new_ret) != self) { - return new_ret; + // The instance returned by __new__() becomes the new object + o = MP_OBJ_TO_PTR(new_ret); } - o = MP_OBJ_TO_PTR(new_ret); - // now call Python class __init__ function with all args + // This method has a chance to call super().__init__() to construct a + // possible native base class. init_fn[0] = init_fn[1] = MP_OBJ_NULL; lookup.obj = o; lookup.attr = MP_QSTR___init__; @@ -329,6 +352,12 @@ mp_obj_t mp_obj_instance_make_new(const mp_obj_type_t *self, size_t n_args, size } + // If the type had a native base that was not explicitly initialised + // (constructed) by the Python __init__() method then construct it now. + if (native_base != NULL && o->subobj[0] == MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj)) { + o->subobj[0] = native_base->make_new(native_base, n_args, n_kw, args); + } + return MP_OBJ_FROM_PTR(o); } @@ -420,7 +449,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = { [MP_BINARY_OP_LESS_EQUAL] = MP_QSTR___le__, [MP_BINARY_OP_MORE_EQUAL] = MP_QSTR___ge__, // MP_BINARY_OP_NOT_EQUAL, // a != b calls a == b and inverts result - [MP_BINARY_OP_IN] = MP_QSTR___contains__, + [MP_BINARY_OP_CONTAINS] = MP_QSTR___contains__, // All inplace methods are optional, and normal methods will be used // as a fallback. @@ -946,21 +975,21 @@ STATIC void type_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { if (self->locals_dict != NULL) { assert(self->locals_dict->base.type == &mp_type_dict); // MicroPython restriction, for now mp_map_t *locals_map = &self->locals_dict->map; + if (locals_map->is_fixed) { + // can't apply delete/store to a fixed map + return; + } if (dest[1] == MP_OBJ_NULL) { // delete attribute mp_map_elem_t *elem = mp_map_lookup(locals_map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_REMOVE_IF_FOUND); - // note that locals_map may be in ROM, so remove will fail in that case if (elem != NULL) { dest[0] = MP_OBJ_NULL; // indicate success } } else { // store attribute mp_map_elem_t *elem = mp_map_lookup(locals_map, MP_OBJ_NEW_QSTR(attr), MP_MAP_LOOKUP_ADD_IF_NOT_FOUND); - // note that locals_map may be in ROM, so add will fail in that case - if (elem != NULL) { - elem->value = dest[1]; - dest[0] = MP_OBJ_NULL; // indicate success - } + elem->value = dest[1]; + dest[0] = MP_OBJ_NULL; // indicate success } } } @@ -983,12 +1012,12 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) // TODO might need to make a copy of locals_dict; at least that's how CPython does it // Basic validation of base classes - size_t len; - mp_obj_t *items; - mp_obj_tuple_get(bases_tuple, &len, &items); - for (size_t i = 0; i < len; i++) { - assert(MP_OBJ_IS_TYPE(items[i], &mp_type_type)); - mp_obj_type_t *t = MP_OBJ_TO_PTR(items[i]); + size_t bases_len; + mp_obj_t *bases_items; + mp_obj_tuple_get(bases_tuple, &bases_len, &bases_items); + for (size_t i = 0; i < bases_len; i++) { + assert(MP_OBJ_IS_TYPE(bases_items[i], &mp_type_type)); + mp_obj_type_t *t = MP_OBJ_TO_PTR(bases_items[i]); // TODO: Verify with CPy, tested on function type if (t->make_new == NULL) { if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { @@ -1014,17 +1043,21 @@ mp_obj_t mp_obj_new_type(qstr name, mp_obj_t bases_tuple, mp_obj_t locals_dict) //o->iternext = ; not implemented o->buffer_p.get_buffer = instance_get_buffer; - if (len > 0) { + if (bases_len > 0) { // Inherit protocol from a base class. This allows to define an // abstract base class which would translate C-level protocol to // Python method calls, and any subclass inheriting from it will // support this feature. - o->protocol = ((mp_obj_type_t*)MP_OBJ_TO_PTR(items[0]))->protocol; + o->protocol = ((mp_obj_type_t*)MP_OBJ_TO_PTR(bases_items[0]))->protocol; - if (len >= 2) { + if (bases_len >= 2) { + #if MICROPY_MULTIPLE_INHERITANCE o->parent = MP_OBJ_TO_PTR(bases_tuple); + #else + mp_raise_NotImplementedError("multiple inheritance not supported"); + #endif } else { - o->parent = MP_OBJ_TO_PTR(items[0]); + o->parent = MP_OBJ_TO_PTR(bases_items[0]); } } @@ -1099,26 +1132,48 @@ STATIC void super_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { .is_type = false, }; + // Allow a call super().__init__() to reach any native base classes + if (attr == MP_QSTR___init__) { + lookup.meth_offset = offsetof(mp_obj_type_t, make_new); + } + if (type->parent == NULL) { // no parents, do nothing + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)type->parent)->type == &mp_type_tuple) { const mp_obj_tuple_t *parent_tuple = type->parent; size_t len = parent_tuple->len; const mp_obj_t *items = parent_tuple->items; for (size_t i = 0; i < len; i++) { assert(MP_OBJ_IS_TYPE(items[i], &mp_type_type)); + if (MP_OBJ_TO_PTR(items[i]) == &mp_type_object) { + // The "object" type will be searched at the end of this function, + // and we don't want to lookup native methods in object. + continue; + } mp_obj_class_lookup(&lookup, (mp_obj_type_t*)MP_OBJ_TO_PTR(items[i])); if (dest[0] != MP_OBJ_NULL) { - return; + break; } } - } else { + #endif + } else if (type->parent != &mp_type_object) { mp_obj_class_lookup(&lookup, type->parent); - if (dest[0] != MP_OBJ_NULL) { - return; - } } + if (dest[0] != MP_OBJ_NULL) { + if (dest[0] == MP_OBJ_SENTINEL) { + // Looked up native __init__ so defer to it + dest[0] = MP_OBJ_FROM_PTR(&native_base_init_wrapper_obj); + dest[1] = self->obj; + } + return; + } + + // Reset meth_offset so we don't look up any native methods in object, + // because object never takes up the native base-class slot. + lookup.meth_offset = 0; + mp_obj_class_lookup(&lookup, &mp_type_object); } @@ -1158,6 +1213,7 @@ bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo) { if (self->parent == NULL) { // type has no parents return false; + #if MICROPY_MULTIPLE_INHERITANCE } else if (((mp_obj_base_t*)self->parent)->type == &mp_type_tuple) { // get the base objects (they should be type objects) const mp_obj_tuple_t *parent_tuple = self->parent; @@ -1173,6 +1229,7 @@ bool mp_obj_is_subclass_fast(mp_const_obj_t object, mp_const_obj_t classinfo) { // search last base (simple tail recursion elimination) object = *item; + #endif } else { // type has 1 parent object = MP_OBJ_FROM_PTR(self->parent); diff --git a/python/src/py/objtype.h b/python/src/py/objtype.h index 52419f3cd..1f4313084 100644 --- a/python/src/py/objtype.h +++ b/python/src/py/objtype.h @@ -37,6 +37,11 @@ typedef struct _mp_obj_instance_t { // TODO maybe cache __getattr__ and __setattr__ for efficient lookup of them } mp_obj_instance_t; +#if MICROPY_CPYTHON_COMPAT +// this is needed for object.__new__ +mp_obj_instance_t *mp_obj_new_instance(const mp_obj_type_t *cls, const mp_obj_type_t **native_base); +#endif + // this needs to be exposed for MICROPY_OPT_CACHE_MAP_LOOKUP_IN_BYTECODE to work void mp_obj_instance_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest); diff --git a/python/src/py/opmethods.c b/python/src/py/opmethods.c index 1200ba39e..247fa5bbc 100644 --- a/python/src/py/opmethods.c +++ b/python/src/py/opmethods.c @@ -24,7 +24,7 @@ * THE SOFTWARE. */ -#include "py/runtime0.h" +#include "py/obj.h" #include "py/builtin.h" STATIC mp_obj_t op_getitem(mp_obj_t self_in, mp_obj_t key_in) { @@ -47,6 +47,6 @@ MP_DEFINE_CONST_FUN_OBJ_2(mp_op_delitem_obj, op_delitem); STATIC mp_obj_t op_contains(mp_obj_t lhs_in, mp_obj_t rhs_in) { mp_obj_type_t *type = mp_obj_get_type(lhs_in); - return type->binary_op(MP_BINARY_OP_IN, lhs_in, rhs_in); + return type->binary_op(MP_BINARY_OP_CONTAINS, lhs_in, rhs_in); } MP_DEFINE_CONST_FUN_OBJ_2(mp_op_contains_obj, op_contains); diff --git a/python/src/py/parse.c b/python/src/py/parse.c index 8c51b0349..8c1286492 100644 --- a/python/src/py/parse.c +++ b/python/src/py/parse.c @@ -3,7 +3,7 @@ * * The MIT License (MIT) * - * Copyright (c) 2013-2015 Damien P. George + * Copyright (c) 2013-2017 Damien P. George * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -58,15 +58,6 @@ // (un)comment to use rule names; for debugging //#define USE_RULE_NAME (1) -typedef struct _rule_t { - byte rule_id; - byte act; -#ifdef USE_RULE_NAME - const char *rule_name; -#endif - uint16_t arg[]; -} rule_t; - enum { // define rules with a compile function #define DEF_RULE(rule, comp, kind, ...) RULE_##rule, @@ -84,6 +75,8 @@ enum { #undef DEF_RULE_NC }; +// Define an array of actions corresponding to each rule +STATIC const uint8_t rule_act_table[] = { #define or(n) (RULE_ACT_OR | n) #define and(n) (RULE_ACT_AND | n) #define and_ident(n) (RULE_ACT_AND | n | RULE_ACT_ALLOW_IDENT) @@ -91,45 +84,129 @@ enum { #define one_or_more (RULE_ACT_LIST | 2) #define list (RULE_ACT_LIST | 1) #define list_with_end (RULE_ACT_LIST | 3) -#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) -#define rule(r) (RULE_ARG_RULE | RULE_##r) -#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) -#ifdef USE_RULE_NAME -#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; -#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, #rule, { __VA_ARGS__ } }; -#else -#define DEF_RULE(rule, comp, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; -#define DEF_RULE_NC(rule, kind, ...) static const rule_t rule_##rule = { RULE_##rule, kind, { __VA_ARGS__ } }; -#endif -#include "py/grammar.h" -#undef or -#undef and -#undef list -#undef list_with_end -#undef tok -#undef rule -#undef opt_rule -#undef one_or_more -#undef DEF_RULE -#undef DEF_RULE_NC -STATIC const rule_t *const rules[] = { -// define rules with a compile function -#define DEF_RULE(rule, comp, kind, ...) &rule_##rule, +#define DEF_RULE(rule, comp, kind, ...) kind, #define DEF_RULE_NC(rule, kind, ...) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC - NULL, // RULE_const_object -// define rules without a compile function + 0, // RULE_const_object + #define DEF_RULE(rule, comp, kind, ...) -#define DEF_RULE_NC(rule, kind, ...) &rule_##rule, +#define DEF_RULE_NC(rule, kind, ...) kind, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + +#undef or +#undef and +#undef and_ident +#undef and_blank +#undef one_or_more +#undef list +#undef list_with_end +}; + +// Define the argument data for each rule, as a combined array +STATIC const uint16_t rule_arg_combined_table[] = { +#define tok(t) (RULE_ARG_TOK | MP_TOKEN_##t) +#define rule(r) (RULE_ARG_RULE | RULE_##r) +#define opt_rule(r) (RULE_ARG_OPT_RULE | RULE_##r) + +#define DEF_RULE(rule, comp, kind, ...) __VA_ARGS__, +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) __VA_ARGS__, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + +#undef tok +#undef rule +#undef opt_rule +}; + +// Macro to create a list of N identifiers where N is the number of variable arguments to the macro +#define RULE_EXPAND(x) x +#define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule)) +#define RULE_PADDING2(rule, ...) RULE_EXPAND(RULE_PADDING3(rule, __VA_ARGS__)) +#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, ...) __VA_ARGS__ +#define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, + +// Use an enum to create constants specifying how much room a rule takes in rule_arg_combined_table +enum { +#define DEF_RULE(rule, comp, kind, ...) RULE_PADDING(rule, __VA_ARGS__) +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_PADDING(rule, __VA_ARGS__) #include "py/grammar.h" #undef DEF_RULE #undef DEF_RULE_NC }; +// Macro to compute the start of a rule in rule_arg_combined_table +#define RULE_ARG_OFFSET(rule, ...) RULE_ARG_OFFSET2(rule, __VA_ARGS__, RULE_ARG_OFFSET_IDS(rule)) +#define RULE_ARG_OFFSET2(rule, ...) RULE_EXPAND(RULE_ARG_OFFSET3(rule, __VA_ARGS__)) +#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) _13 +#define RULE_ARG_OFFSET_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r, + +// Use the above enum values to create a table of offsets for each rule's arg +// data, which indexes rule_arg_combined_table. The offsets require 9 bits of +// storage but only the lower 8 bits are stored here. The 9th bit is computed +// in get_rule_arg using the FIRST_RULE_WITH_OFFSET_ABOVE_255 constant. +STATIC const uint8_t rule_arg_offset_table[] = { +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff, +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + 0, // RULE_const_object +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) & 0xff, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +}; + +// Define a constant that's used to determine the 9th bit of the values in rule_arg_offset_table +static const size_t FIRST_RULE_WITH_OFFSET_ABOVE_255 = +#define DEF_RULE(rule, comp, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) >= 0x100 ? RULE_##rule : +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) RULE_ARG_OFFSET(rule, __VA_ARGS__) >= 0x100 ? RULE_##rule : +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +0; + +#if USE_RULE_NAME +// Define an array of rule names corresponding to each rule +STATIC const char *const rule_name_table[] = { +#define DEF_RULE(rule, comp, kind, ...) #rule, +#define DEF_RULE_NC(rule, kind, ...) +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC + "", // RULE_const_object +#define DEF_RULE(rule, comp, kind, ...) +#define DEF_RULE_NC(rule, kind, ...) #rule, +#include "py/grammar.h" +#undef DEF_RULE +#undef DEF_RULE_NC +}; +#endif + typedef struct _rule_stack_t { size_t src_line : 8 * sizeof(size_t) - 8; // maximum bits storing source line number size_t rule_id : 8; // this must be large enough to fit largest rule number @@ -164,6 +241,14 @@ typedef struct _parser_t { #endif } parser_t; +STATIC const uint16_t *get_rule_arg(uint8_t r_id) { + size_t off = rule_arg_offset_table[r_id]; + if (r_id >= FIRST_RULE_WITH_OFFSET_ABOVE_255) { + off |= 0x100; + } + return &rule_arg_combined_table[off]; +} + STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) { // use a custom memory allocator to store parse nodes sequentially in large chunks @@ -205,7 +290,7 @@ STATIC void *parser_alloc(parser_t *parser, size_t num_bytes) { return ret; } -STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t arg_i) { +STATIC void push_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t arg_i) { if (parser->rule_stack_top >= parser->rule_stack_alloc) { rule_stack_t *rs = m_renew(rule_stack_t, parser->rule_stack, parser->rule_stack_alloc, parser->rule_stack_alloc + MICROPY_ALLOC_PARSE_RULE_INC); parser->rule_stack = rs; @@ -213,21 +298,22 @@ STATIC void push_rule(parser_t *parser, size_t src_line, const rule_t *rule, siz } rule_stack_t *rs = &parser->rule_stack[parser->rule_stack_top++]; rs->src_line = src_line; - rs->rule_id = rule->rule_id; + rs->rule_id = rule_id; rs->arg_i = arg_i; } STATIC void push_rule_from_arg(parser_t *parser, size_t arg) { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE || (arg & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE); size_t rule_id = arg & RULE_ARG_ARG_MASK; - push_rule(parser, parser->lexer->tok_line, rules[rule_id], 0); + push_rule(parser, parser->lexer->tok_line, rule_id, 0); } -STATIC void pop_rule(parser_t *parser, const rule_t **rule, size_t *arg_i, size_t *src_line) { +STATIC uint8_t pop_rule(parser_t *parser, size_t *arg_i, size_t *src_line) { parser->rule_stack_top -= 1; - *rule = rules[parser->rule_stack[parser->rule_stack_top].rule_id]; + uint8_t rule_id = parser->rule_stack[parser->rule_stack_top].rule_id; *arg_i = parser->rule_stack[parser->rule_stack_top].arg_i; *src_line = parser->rule_stack[parser->rule_stack_top].src_line; + return rule_id; } bool mp_parse_node_is_const_false(mp_parse_node_t pn) { @@ -313,11 +399,11 @@ void mp_parse_node_print(mp_parse_node_t pn, size_t indent) { #endif } else { size_t n = MP_PARSE_NODE_STRUCT_NUM_NODES(pns); -#ifdef USE_RULE_NAME - printf("%s(%u) (n=%u)\n", rules[MP_PARSE_NODE_STRUCT_KIND(pns)]->rule_name, (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n); -#else + #if USE_RULE_NAME + printf("%s(%u) (n=%u)\n", rule_name_table[MP_PARSE_NODE_STRUCT_KIND(pns)], (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n); + #else printf("rule(%u) (n=%u)\n", (uint)MP_PARSE_NODE_STRUCT_KIND(pns), (uint)n); -#endif + #endif for (size_t i = 0; i < n; i++) { mp_parse_node_print(pns->nodes[i], indent + 2); } @@ -369,7 +455,19 @@ STATIC mp_parse_node_t make_node_const_object(parser_t *parser, size_t src_line, return (mp_parse_node_t)pn; } -STATIC void push_result_token(parser_t *parser, const rule_t *rule) { +STATIC mp_parse_node_t mp_parse_node_new_small_int_checked(parser_t *parser, mp_obj_t o_val) { + (void)parser; + mp_int_t val = MP_OBJ_SMALL_INT_VALUE(o_val); + #if MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D + // A parse node is only 32-bits and the small-int value must fit in 31-bits + if (((val ^ (val << 1)) & 0xffffffff80000000) != 0) { + return make_node_const_object(parser, 0, o_val); + } + #endif + return mp_parse_node_new_small_int(val); +} + +STATIC void push_result_token(parser_t *parser, uint8_t rule_id) { mp_parse_node_t pn; mp_lexer_t *lex = parser->lexer; if (lex->tok_kind == MP_TOKEN_NAME) { @@ -377,10 +475,10 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { #if MICROPY_COMP_CONST // if name is a standalone identifier, look it up in the table of dynamic constants mp_map_elem_t *elem; - if (rule->rule_id == RULE_atom + if (rule_id == RULE_atom && (elem = mp_map_lookup(&parser->consts, MP_OBJ_NEW_QSTR(id), MP_MAP_LOOKUP)) != NULL) { if (MP_OBJ_IS_SMALL_INT(elem->value)) { - pn = mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(elem->value)); + pn = mp_parse_node_new_small_int_checked(parser, elem->value); } else { pn = make_node_const_object(parser, lex->tok_line, elem->value); } @@ -388,13 +486,13 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id); } #else - (void)rule; + (void)rule_id; pn = mp_parse_node_new_leaf(MP_PARSE_NODE_ID, id); #endif } else if (lex->tok_kind == MP_TOKEN_INTEGER) { mp_obj_t o = mp_parse_num_integer(lex->vstr.buf, lex->vstr.len, 0, lex); if (MP_OBJ_IS_SMALL_INT(o)) { - pn = mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(o)); + pn = mp_parse_node_new_small_int_checked(parser, o); } else { pn = make_node_const_object(parser, lex->tok_line, o); } @@ -417,7 +515,7 @@ STATIC void push_result_token(parser_t *parser, const rule_t *rule) { pn = mp_parse_node_new_leaf(lex->tok_kind == MP_TOKEN_STRING ? MP_PARSE_NODE_STRING : MP_PARSE_NODE_BYTES, qst); } else { // not interned, make a node holding a pointer to the string/bytes object - mp_obj_t o = mp_obj_new_str_of_type( + mp_obj_t o = mp_obj_new_str_copy( lex->tok_kind == MP_TOKEN_STRING ? &mp_type_str : &mp_type_bytes, (const byte*)lex->vstr.buf, lex->vstr.len); pn = make_node_const_object(parser, lex->tok_line, o); @@ -442,12 +540,12 @@ STATIC const mp_rom_map_elem_t mp_constants_table[] = { STATIC MP_DEFINE_CONST_MAP(mp_constants_map, mp_constants_table); #endif -STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t num_args); +STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args); #if MICROPY_COMP_CONST_FOLDING -STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t *num_args) { - if (rule->rule_id == RULE_or_test - || rule->rule_id == RULE_and_test) { +STATIC bool fold_logical_constants(parser_t *parser, uint8_t rule_id, size_t *num_args) { + if (rule_id == RULE_or_test + || rule_id == RULE_and_test) { // folding for binary logical ops: or and size_t copy_to = *num_args; for (size_t i = copy_to; i > 0;) { @@ -457,7 +555,7 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t // always need to keep the last value break; } - if (rule->rule_id == RULE_or_test) { + if (rule_id == RULE_or_test) { if (mp_parse_node_is_const_true(pn)) { // break; @@ -484,7 +582,7 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t // we did a complete folding if there's only 1 arg left return *num_args == 1; - } else if (rule->rule_id == RULE_not_test_2) { + } else if (rule_id == RULE_not_test_2) { // folding for unary logical op: not mp_parse_node_t pn = peek_result(parser, 0); if (mp_parse_node_is_const_false(pn)) { @@ -502,23 +600,23 @@ STATIC bool fold_logical_constants(parser_t *parser, const rule_t *rule, size_t return false; } -STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args) { +STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) { // this code does folding of arbitrary integer expressions, eg 1 + 2 * 3 + 4 // it does not do partial folding, eg 1 + 2 + x -> 3 + x mp_obj_t arg0; - if (rule->rule_id == RULE_expr - || rule->rule_id == RULE_xor_expr - || rule->rule_id == RULE_and_expr) { + if (rule_id == RULE_expr + || rule_id == RULE_xor_expr + || rule_id == RULE_and_expr) { // folding for binary ops: | ^ & mp_parse_node_t pn = peek_result(parser, num_args - 1); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { return false; } mp_binary_op_t op; - if (rule->rule_id == RULE_expr) { + if (rule_id == RULE_expr) { op = MP_BINARY_OP_OR; - } else if (rule->rule_id == RULE_xor_expr) { + } else if (rule_id == RULE_xor_expr) { op = MP_BINARY_OP_XOR; } else { op = MP_BINARY_OP_AND; @@ -531,9 +629,9 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args } arg0 = mp_binary_op(op, arg0, arg1); } - } else if (rule->rule_id == RULE_shift_expr - || rule->rule_id == RULE_arith_expr - || rule->rule_id == RULE_term) { + } else if (rule_id == RULE_shift_expr + || rule_id == RULE_arith_expr + || rule_id == RULE_term) { // folding for binary ops: << >> + - * / % // mp_parse_node_t pn = peek_result(parser, num_args - 1); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { @@ -577,7 +675,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args } arg0 = mp_binary_op(op, arg0, arg1); } - } else if (rule->rule_id == RULE_factor_2) { + } else if (rule_id == RULE_factor_2) { // folding for unary ops: + - ~ mp_parse_node_t pn = peek_result(parser, 0); if (!mp_parse_node_get_int_maybe(pn, &arg0)) { @@ -596,7 +694,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args arg0 = mp_unary_op(op, arg0); #if MICROPY_COMP_CONST - } else if (rule->rule_id == RULE_expr_stmt) { + } else if (rule_id == RULE_expr_stmt) { mp_parse_node_t pn1 = peek_result(parser, 0); if (!MP_PARSE_NODE_IS_NULL(pn1) && !(MP_PARSE_NODE_IS_STRUCT_KIND(pn1, RULE_expr_stmt_augassign) @@ -635,7 +733,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args if (qstr_str(id)[0] == '_') { pop_result(parser); // pop const(value) pop_result(parser); // pop id - push_result_rule(parser, 0, rules[RULE_pass_stmt], 0); // replace with "pass" + push_result_rule(parser, 0, RULE_pass_stmt, 0); // replace with "pass" return true; } @@ -651,7 +749,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args #endif #if MICROPY_COMP_MODULE_CONST - } else if (rule->rule_id == RULE_atom_expr_normal) { + } else if (rule_id == RULE_atom_expr_normal) { mp_parse_node_t pn0 = peek_result(parser, 1); mp_parse_node_t pn1 = peek_result(parser, 0); if (!(MP_PARSE_NODE_IS_ID(pn0) @@ -686,7 +784,7 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args pop_result(parser); } if (MP_OBJ_IS_SMALL_INT(arg0)) { - push_result_node(parser, mp_parse_node_new_small_int(MP_OBJ_SMALL_INT_VALUE(arg0))); + push_result_node(parser, mp_parse_node_new_small_int_checked(parser, arg0)); } else { // TODO reuse memory for parse node struct? push_result_node(parser, make_node_const_object(parser, 0, arg0)); @@ -696,9 +794,9 @@ STATIC bool fold_constants(parser_t *parser, const rule_t *rule, size_t num_args } #endif -STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *rule, size_t num_args) { +STATIC void push_result_rule(parser_t *parser, size_t src_line, uint8_t rule_id, size_t num_args) { // optimise away parenthesis around an expression if possible - if (rule->rule_id == RULE_atom_paren) { + if (rule_id == RULE_atom_paren) { // there should be just 1 arg for this rule mp_parse_node_t pn = peek_result(parser, 0); if (MP_PARSE_NODE_IS_NULL(pn)) { @@ -712,11 +810,11 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *ru } #if MICROPY_COMP_CONST_FOLDING - if (fold_logical_constants(parser, rule, &num_args)) { + if (fold_logical_constants(parser, rule_id, &num_args)) { // we folded this rule so return straight away return; } - if (fold_constants(parser, rule, num_args)) { + if (fold_constants(parser, rule_id, num_args)) { // we folded this rule so return straight away return; } @@ -724,7 +822,7 @@ STATIC void push_result_rule(parser_t *parser, size_t src_line, const rule_t *ru mp_parse_node_struct_t *pn = parser_alloc(parser, sizeof(mp_parse_node_struct_t) + sizeof(mp_parse_node_t) * num_args); pn->source_line = src_line; - pn->kind_num_nodes = (rule->rule_id & 0xff) | (num_args << 8); + pn->kind_num_nodes = (rule_id & 0xff) | (num_args << 8); for (size_t i = num_args; i > 0; i--) { pn->nodes[i - 1] = pop_result(parser); } @@ -761,14 +859,11 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { case MP_PARSE_EVAL_INPUT: top_level_rule = RULE_eval_input; break; default: top_level_rule = RULE_file_input; } - push_rule(&parser, lex->tok_line, rules[top_level_rule], 0); + push_rule(&parser, lex->tok_line, top_level_rule, 0); // parse! - size_t n, i; // state for the current rule - size_t rule_src_line; // source line for the first token matched by the current rule bool backtrack = false; - const rule_t *rule = NULL; for (;;) { next_rule: @@ -776,19 +871,24 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { break; } - pop_rule(&parser, &rule, &i, &rule_src_line); - n = rule->act & RULE_ACT_ARG_MASK; + // Pop the next rule to process it + size_t i; // state for the current rule + size_t rule_src_line; // source line for the first token matched by the current rule + uint8_t rule_id = pop_rule(&parser, &i, &rule_src_line); + uint8_t rule_act = rule_act_table[rule_id]; + const uint16_t *rule_arg = get_rule_arg(rule_id); + size_t n = rule_act & RULE_ACT_ARG_MASK; - /* + #if 0 // debugging - printf("depth=%d ", parser.rule_stack_top); + printf("depth=" UINT_FMT " ", parser.rule_stack_top); for (int j = 0; j < parser.rule_stack_top; ++j) { printf(" "); } - printf("%s n=%d i=%d bt=%d\n", rule->rule_name, n, i, backtrack); - */ + printf("%s n=" UINT_FMT " i=" UINT_FMT " bt=%d\n", rule_name_table[rule_id], n, i, backtrack); + #endif - switch (rule->act & RULE_ACT_KIND_MASK) { + switch (rule_act & RULE_ACT_KIND_MASK) { case RULE_ACT_OR: if (i > 0 && !backtrack) { goto next_rule; @@ -796,19 +896,19 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { backtrack = false; } for (; i < n; ++i) { - uint16_t kind = rule->arg[i] & RULE_ARG_KIND_MASK; + uint16_t kind = rule_arg[i] & RULE_ARG_KIND_MASK; if (kind == RULE_ARG_TOK) { - if (lex->tok_kind == (rule->arg[i] & RULE_ARG_ARG_MASK)) { - push_result_token(&parser, rule); + if (lex->tok_kind == (rule_arg[i] & RULE_ARG_ARG_MASK)) { + push_result_token(&parser, rule_id); mp_lexer_to_next(lex); goto next_rule; } } else { assert(kind == RULE_ARG_RULE); if (i + 1 < n) { - push_rule(&parser, rule_src_line, rule, i + 1); // save this or-rule + push_rule(&parser, rule_src_line, rule_id, i + 1); // save this or-rule } - push_rule_from_arg(&parser, rule->arg[i]); // push child of or-rule + push_rule_from_arg(&parser, rule_arg[i]); // push child of or-rule goto next_rule; } } @@ -820,7 +920,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // failed, backtrack if we can, else syntax error if (backtrack) { assert(i > 0); - if ((rule->arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { + if ((rule_arg[i - 1] & RULE_ARG_KIND_MASK) == RULE_ARG_OPT_RULE) { // an optional rule that failed, so continue with next arg push_result_node(&parser, MP_PARSE_NODE_NULL); backtrack = false; @@ -837,13 +937,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // progress through the rule for (; i < n; ++i) { - if ((rule->arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + if ((rule_arg[i] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { // need to match a token - mp_token_kind_t tok_kind = rule->arg[i] & RULE_ARG_ARG_MASK; + mp_token_kind_t tok_kind = rule_arg[i] & RULE_ARG_ARG_MASK; if (lex->tok_kind == tok_kind) { // matched token if (tok_kind == MP_TOKEN_NAME) { - push_result_token(&parser, rule); + push_result_token(&parser, rule_id); } mp_lexer_to_next(lex); } else { @@ -858,8 +958,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } } else { - push_rule(&parser, rule_src_line, rule, i + 1); // save this and-rule - push_rule_from_arg(&parser, rule->arg[i]); // push child of and-rule + push_rule(&parser, rule_src_line, rule_id, i + 1); // save this and-rule + push_rule_from_arg(&parser, rule_arg[i]); // push child of and-rule goto next_rule; } } @@ -870,7 +970,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { #if !MICROPY_ENABLE_DOC_STRING // this code discards lonely statements, such as doc strings - if (input_kind != MP_PARSE_SINGLE_INPUT && rule->rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { + if (input_kind != MP_PARSE_SINGLE_INPUT && rule_id == RULE_expr_stmt && peek_result(&parser, 0) == MP_PARSE_NODE_NULL) { mp_parse_node_t p = peek_result(&parser, 1); if ((MP_PARSE_NODE_IS_LEAF(p) && !MP_PARSE_NODE_IS_ID(p)) || MP_PARSE_NODE_IS_STRUCT_KIND(p, RULE_const_object)) { @@ -879,7 +979,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // Pushing the "pass" rule here will overwrite any RULE_const_object // entry that was on the result stack, allowing the GC to reclaim // the memory from the const object when needed. - push_result_rule(&parser, rule_src_line, rules[RULE_pass_stmt], 0); + push_result_rule(&parser, rule_src_line, RULE_pass_stmt, 0); break; } } @@ -890,8 +990,8 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { size_t num_not_nil = 0; for (size_t x = n; x > 0;) { --x; - if ((rule->arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { - mp_token_kind_t tok_kind = rule->arg[x] & RULE_ARG_ARG_MASK; + if ((rule_arg[x] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + mp_token_kind_t tok_kind = rule_arg[x] & RULE_ARG_ARG_MASK; if (tok_kind == MP_TOKEN_NAME) { // only tokens which were names are pushed to stack i += 1; @@ -906,7 +1006,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } - if (num_not_nil == 1 && (rule->act & RULE_ACT_ALLOW_IDENT)) { + if (num_not_nil == 1 && (rule_act & RULE_ACT_ALLOW_IDENT)) { // this rule has only 1 argument and should not be emitted mp_parse_node_t pn = MP_PARSE_NODE_NULL; for (size_t x = 0; x < i; ++x) { @@ -919,19 +1019,19 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } else { // this rule must be emitted - if (rule->act & RULE_ACT_ADD_BLANK) { + if (rule_act & RULE_ACT_ADD_BLANK) { // and add an extra blank node at the end (used by the compiler to store data) push_result_node(&parser, MP_PARSE_NODE_NULL); i += 1; } - push_result_rule(&parser, rule_src_line, rule, i); + push_result_rule(&parser, rule_src_line, rule_id, i); } break; } default: { - assert((rule->act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST); + assert((rule_act & RULE_ACT_KIND_MASK) == RULE_ACT_LIST); // n=2 is: item item* // n=1 is: item (sep item)* @@ -969,13 +1069,13 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { for (;;) { - size_t arg = rule->arg[i & 1 & n]; + size_t arg = rule_arg[i & 1 & n]; if ((arg & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { if (lex->tok_kind == (arg & RULE_ARG_ARG_MASK)) { if (i & 1 & n) { // separators which are tokens are not pushed to result stack } else { - push_result_token(&parser, rule); + push_result_token(&parser, rule_id); } mp_lexer_to_next(lex); // got element of list, so continue parsing list @@ -988,7 +1088,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { } } else { assert((arg & RULE_ARG_KIND_MASK) == RULE_ARG_RULE); - push_rule(&parser, rule_src_line, rule, i + 1); // save this list-rule + push_rule(&parser, rule_src_line, rule_id, i + 1); // save this list-rule push_rule_from_arg(&parser, arg); // push child of list-rule goto next_rule; } @@ -998,7 +1098,7 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // compute number of elements in list, result in i i -= 1; - if ((n & 1) && (rule->arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { + if ((n & 1) && (rule_arg[1] & RULE_ARG_KIND_MASK) == RULE_ARG_TOK) { // don't count separators when they are tokens i = (i + 1) / 2; } @@ -1007,12 +1107,12 @@ mp_parse_tree_t mp_parse(mp_lexer_t *lex, mp_parse_input_kind_t input_kind) { // list matched single item if (had_trailing_sep) { // if there was a trailing separator, make a list of a single item - push_result_rule(&parser, rule_src_line, rule, i); + push_result_rule(&parser, rule_src_line, rule_id, i); } else { // just leave single item on stack (ie don't wrap in a list) } } else { - push_result_rule(&parser, rule_src_line, rule, i); + push_result_rule(&parser, rule_src_line, rule_id, i); } break; } diff --git a/python/src/py/parsenum.c b/python/src/py/parsenum.c index b62029f7c..124489c66 100644 --- a/python/src/py/parsenum.c +++ b/python/src/py/parsenum.c @@ -170,6 +170,19 @@ typedef enum { mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool force_complex, mp_lexer_t *lex) { #if MICROPY_PY_BUILTINS_FLOAT + +// DEC_VAL_MAX only needs to be rough and is used to retain precision while not overflowing +// SMALL_NORMAL_VAL is the smallest power of 10 that is still a normal float +#if MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_FLOAT +#define DEC_VAL_MAX 1e20F +#define SMALL_NORMAL_VAL (1e-37F) +#define SMALL_NORMAL_EXP (-37) +#elif MICROPY_FLOAT_IMPL == MICROPY_FLOAT_IMPL_DOUBLE +#define DEC_VAL_MAX 1e200 +#define SMALL_NORMAL_VAL (1e-307) +#define SMALL_NORMAL_EXP (-307) +#endif + const char *top = str + len; mp_float_t dec_val = 0; bool dec_neg = false; @@ -214,8 +227,8 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool // string should be a decimal number parse_dec_in_t in = PARSE_DEC_IN_INTG; bool exp_neg = false; - mp_float_t frac_mult = 0.1; mp_int_t exp_val = 0; + mp_int_t exp_extra = 0; while (str < top) { mp_uint_t dig = *str++; if ('0' <= dig && dig <= '9') { @@ -223,11 +236,18 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool if (in == PARSE_DEC_IN_EXP) { exp_val = 10 * exp_val + dig; } else { - if (in == PARSE_DEC_IN_FRAC) { - dec_val += dig * frac_mult; - frac_mult *= MICROPY_FLOAT_CONST(0.1); - } else { + if (dec_val < DEC_VAL_MAX) { + // dec_val won't overflow so keep accumulating dec_val = 10 * dec_val + dig; + if (in == PARSE_DEC_IN_FRAC) { + --exp_extra; + } + } else { + // dec_val might overflow and we anyway can't represent more digits + // of precision, so ignore the digit and just adjust the exponent + if (in == PARSE_DEC_IN_INTG) { + ++exp_extra; + } } } } else if (in == PARSE_DEC_IN_INTG && dig == '.') { @@ -260,7 +280,12 @@ mp_obj_t mp_parse_num_decimal(const char *str, size_t len, bool allow_imag, bool exp_val = -exp_val; } - // apply the exponent + // apply the exponent, making sure it's not a subnormal value + exp_val += exp_extra; + if (exp_val < SMALL_NORMAL_EXP) { + exp_val -= SMALL_NORMAL_EXP; + dec_val *= SMALL_NORMAL_VAL; + } dec_val *= MICROPY_FLOAT_C_FUN(pow)(10, exp_val); } diff --git a/python/src/py/persistentcode.c b/python/src/py/persistentcode.c index e0bb8f1d6..7113b0dc2 100644 --- a/python/src/py/persistentcode.c +++ b/python/src/py/persistentcode.c @@ -200,7 +200,11 @@ STATIC mp_raw_code_t *load_raw_code(mp_reader_t *reader) { // create raw_code and return it mp_raw_code_t *rc = mp_emit_glue_new_raw_code(); - mp_emit_glue_assign_bytecode(rc, bytecode, bc_len, const_table, + mp_emit_glue_assign_bytecode(rc, bytecode, + #if MICROPY_PERSISTENT_CODE_SAVE || MICROPY_DEBUG_PRINTERS + bc_len, + #endif + const_table, #if MICROPY_PERSISTENT_CODE_SAVE n_obj, n_raw_code, #endif diff --git a/python/src/py/pystack.c b/python/src/py/pystack.c new file mode 100644 index 000000000..552e59d53 --- /dev/null +++ b/python/src/py/pystack.c @@ -0,0 +1,57 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include + +#include "py/runtime.h" + +#if MICROPY_ENABLE_PYSTACK + +void mp_pystack_init(void *start, void *end) { + MP_STATE_THREAD(pystack_start) = start; + MP_STATE_THREAD(pystack_end) = end; + MP_STATE_THREAD(pystack_cur) = start; +} + +void *mp_pystack_alloc(size_t n_bytes) { + n_bytes = (n_bytes + (MICROPY_PYSTACK_ALIGN - 1)) & ~(MICROPY_PYSTACK_ALIGN - 1); + #if MP_PYSTACK_DEBUG + n_bytes += MICROPY_PYSTACK_ALIGN; + #endif + if (MP_STATE_THREAD(pystack_cur) + n_bytes > MP_STATE_THREAD(pystack_end)) { + // out of memory in the pystack + nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, + MP_OBJ_NEW_QSTR(MP_QSTR_pystack_space_exhausted))); + } + void *ptr = MP_STATE_THREAD(pystack_cur); + MP_STATE_THREAD(pystack_cur) += n_bytes; + #if MP_PYSTACK_DEBUG + *(size_t*)(MP_STATE_THREAD(pystack_cur) - MICROPY_PYSTACK_ALIGN) = n_bytes; + #endif + return ptr; +} + +#endif diff --git a/python/src/py/pystack.h b/python/src/py/pystack.h new file mode 100644 index 000000000..82ac3743d --- /dev/null +++ b/python/src/py/pystack.h @@ -0,0 +1,123 @@ +/* + * This file is part of the MicroPython project, http://micropython.org/ + * + * The MIT License (MIT) + * + * Copyright (c) 2017 Damien P. George + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#ifndef MICROPY_INCLUDED_PY_PYSTACK_H +#define MICROPY_INCLUDED_PY_PYSTACK_H + +#include "py/mpstate.h" + +// Enable this debugging option to check that the amount of memory freed is +// consistent with amounts that were previously allocated. +#define MP_PYSTACK_DEBUG (0) + +#if MICROPY_ENABLE_PYSTACK + +void mp_pystack_init(void *start, void *end); +void *mp_pystack_alloc(size_t n_bytes); + +// This function can free multiple continuous blocks at once: just pass the +// pointer to the block that was allocated first and it and all subsequently +// allocated blocks will be freed. +static inline void mp_pystack_free(void *ptr) { + assert((uint8_t*)ptr >= MP_STATE_THREAD(pystack_start)); + assert((uint8_t*)ptr <= MP_STATE_THREAD(pystack_cur)); + #if MP_PYSTACK_DEBUG + size_t n_bytes_to_free = MP_STATE_THREAD(pystack_cur) - (uint8_t*)ptr; + size_t n_bytes = *(size_t*)(MP_STATE_THREAD(pystack_cur) - MICROPY_PYSTACK_ALIGN); + while (n_bytes < n_bytes_to_free) { + n_bytes += *(size_t*)(MP_STATE_THREAD(pystack_cur) - n_bytes - MICROPY_PYSTACK_ALIGN); + } + if (n_bytes != n_bytes_to_free) { + mp_printf(&mp_plat_print, "mp_pystack_free() failed: %u != %u\n", (uint)n_bytes_to_free, + (uint)*(size_t*)(MP_STATE_THREAD(pystack_cur) - MICROPY_PYSTACK_ALIGN)); + assert(0); + } + #endif + MP_STATE_THREAD(pystack_cur) = (uint8_t*)ptr; +} + +static inline void mp_pystack_realloc(void *ptr, size_t n_bytes) { + mp_pystack_free(ptr); + mp_pystack_alloc(n_bytes); +} + +static inline size_t mp_pystack_usage(void) { + return MP_STATE_THREAD(pystack_cur) - MP_STATE_THREAD(pystack_start); +} + +static inline size_t mp_pystack_limit(void) { + return MP_STATE_THREAD(pystack_end) - MP_STATE_THREAD(pystack_start); +} + +#endif + +#if !MICROPY_ENABLE_PYSTACK + +#define mp_local_alloc(n_bytes) alloca(n_bytes) + +static inline void mp_local_free(void *ptr) { + (void)ptr; +} + +static inline void *mp_nonlocal_alloc(size_t n_bytes) { + return m_new(uint8_t, n_bytes); +} + +static inline void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { + return m_renew(uint8_t, ptr, old_n_bytes, new_n_bytes); +} + +static inline void mp_nonlocal_free(void *ptr, size_t n_bytes) { + m_del(uint8_t, ptr, n_bytes); +} + +#else + +static inline void *mp_local_alloc(size_t n_bytes) { + return mp_pystack_alloc(n_bytes); +} + +static inline void mp_local_free(void *ptr) { + mp_pystack_free(ptr); +} + +static inline void *mp_nonlocal_alloc(size_t n_bytes) { + return mp_pystack_alloc(n_bytes); +} + +static inline void *mp_nonlocal_realloc(void *ptr, size_t old_n_bytes, size_t new_n_bytes) { + (void)old_n_bytes; + mp_pystack_realloc(ptr, new_n_bytes); + return ptr; +} + +static inline void mp_nonlocal_free(void *ptr, size_t n_bytes) { + (void)n_bytes; + mp_pystack_free(ptr); +} + +#endif + +#endif // MICROPY_INCLUDED_PY_PYSTACK_H diff --git a/python/src/py/qstr.c b/python/src/py/qstr.c index 95c9b6835..08c3e2505 100644 --- a/python/src/py/qstr.c +++ b/python/src/py/qstr.c @@ -127,14 +127,12 @@ void qstr_init(void) { STATIC const byte *find_qstr(qstr q) { // search pool for this qstr - for (qstr_pool_t *pool = MP_STATE_VM(last_pool); pool != NULL; pool = pool->prev) { - if (q >= pool->total_prev_len) { - return pool->qstrs[q - pool->total_prev_len]; - } + // total_prev_len==0 in the final pool, so the loop will always terminate + qstr_pool_t *pool = MP_STATE_VM(last_pool); + while (q < pool->total_prev_len) { + pool = pool->prev; } - - // not found - return 0; + return pool->qstrs[q - pool->total_prev_len]; } // qstr_mutex must be taken while in this function @@ -243,29 +241,6 @@ qstr qstr_from_strn(const char *str, size_t len) { return q; } -byte *qstr_build_start(size_t len, byte **q_ptr) { - assert(len < (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN))); - *q_ptr = m_new(byte, MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len + 1); - Q_SET_LENGTH(*q_ptr, len); - return Q_GET_DATA(*q_ptr); -} - -qstr qstr_build_end(byte *q_ptr) { - QSTR_ENTER(); - qstr q = qstr_find_strn((const char*)Q_GET_DATA(q_ptr), Q_GET_LENGTH(q_ptr)); - if (q == 0) { - size_t len = Q_GET_LENGTH(q_ptr); - mp_uint_t hash = qstr_compute_hash(Q_GET_DATA(q_ptr), len); - Q_SET_HASH(q_ptr, hash); - q_ptr[MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len] = '\0'; - q = qstr_add(q_ptr); - } else { - m_del(byte, q_ptr, Q_GET_ALLOC(q_ptr)); - } - QSTR_EXIT(); - return q; -} - mp_uint_t qstr_hash(qstr q) { return Q_GET_HASH(find_qstr(q)); } diff --git a/python/src/py/qstr.h b/python/src/py/qstr.h index e2bdcc351..f4375ee0e 100644 --- a/python/src/py/qstr.h +++ b/python/src/py/qstr.h @@ -56,6 +56,7 @@ typedef struct _qstr_pool_t { } qstr_pool_t; #define QSTR_FROM_STR_STATIC(s) (qstr_from_strn((s), strlen(s))) +#define QSTR_TOTAL() (MP_STATE_VM(last_pool)->total_prev_len + MP_STATE_VM(last_pool)->len) void qstr_init(void); @@ -65,9 +66,6 @@ qstr qstr_find_strn(const char *str, size_t str_len); // returns MP_QSTR_NULL if qstr qstr_from_str(const char *str); qstr qstr_from_strn(const char *str, size_t len); -byte *qstr_build_start(size_t len, byte **q_ptr); -qstr qstr_build_end(byte *q_ptr); - mp_uint_t qstr_hash(qstr q); const char *qstr_str(qstr q); size_t qstr_len(qstr q); diff --git a/python/src/py/qstrdefs.h b/python/src/py/qstrdefs.h index 4ded5be08..a60905812 100644 --- a/python/src/py/qstrdefs.h +++ b/python/src/py/qstrdefs.h @@ -40,6 +40,7 @@ Q(/) Q(%#o) Q(%#x) Q({:#b}) +Q( ) Q(\n) Q(maximum recursion depth exceeded) Q() @@ -51,3 +52,7 @@ Q() Q() Q() Q(utf-8) + +#if MICROPY_ENABLE_PYSTACK +Q(pystack exhausted) +#endif diff --git a/python/src/py/repl.c b/python/src/py/repl.c index 7e8922e19..5dce8bbb7 100644 --- a/python/src/py/repl.c +++ b/python/src/py/repl.c @@ -27,6 +27,7 @@ #include #include "py/obj.h" #include "py/runtime.h" +#include "py/builtin.h" #include "py/repl.h" #if MICROPY_HELPER_REPL @@ -136,8 +137,11 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print } } - // begin search in locals dict - mp_obj_dict_t *dict = mp_locals_get(); + size_t nqstr = QSTR_TOTAL(); + + // begin search in outer global dict which is accessed from __main__ + mp_obj_t obj = MP_OBJ_FROM_PTR(&mp_module___main__); + mp_obj_t dest[2]; for (;;) { // get next word in string to complete @@ -148,43 +152,20 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print size_t s_len = str - s_start; if (str < top) { - // a complete word, lookup in current dict - - mp_obj_t obj = MP_OBJ_NULL; - for (size_t i = 0; i < dict->map.alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { - size_t d_len; - const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len); - if (s_len == d_len && strncmp(s_start, d_str, d_len) == 0) { - obj = dict->map.table[i].value; - break; - } - } + // a complete word, lookup in current object + qstr q = qstr_find_strn(s_start, s_len); + if (q == MP_QSTR_NULL) { + // lookup will fail + return 0; } + mp_load_method_protected(obj, q, dest, true); + obj = dest[0]; // attribute, method, or MP_OBJ_NULL if nothing found if (obj == MP_OBJ_NULL) { // lookup failed return 0; } - // found an object of this name; try to get its dict - if (MP_OBJ_IS_TYPE(obj, &mp_type_module)) { - dict = mp_obj_module_get_globals(obj); - } else { - mp_obj_type_t *type; - if (MP_OBJ_IS_TYPE(obj, &mp_type_type)) { - type = MP_OBJ_TO_PTR(obj); - } else { - type = mp_obj_get_type(obj); - } - if (type->locals_dict != NULL && type->locals_dict->base.type == &mp_type_dict) { - dict = type->locals_dict; - } else { - // obj has no dict - return 0; - } - } - // skip '.' to move to next word ++str; @@ -192,14 +173,15 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print // end of string, do completion on this partial name // look for matches - int n_found = 0; const char *match_str = NULL; size_t match_len = 0; - for (size_t i = 0; i < dict->map.alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { - size_t d_len; - const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len); - if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + qstr q_first = 0, q_last = 0; + for (qstr q = MP_QSTR_ + 1; q < nqstr; ++q) { + size_t d_len; + const char *d_str = (const char*)qstr_data(q, &d_len); + if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + mp_load_method_protected(obj, q, dest, true); + if (dest[0] != MP_OBJ_NULL) { if (match_str == NULL) { match_str = d_str; match_len = d_len; @@ -213,13 +195,16 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print } } } - ++n_found; + if (q_first == 0) { + q_first = q; + } + q_last = q; } } } // nothing found - if (n_found == 0) { + if (q_first == 0) { // If there're no better alternatives, and if it's first word // in the line, try to complete "import". if (s_start == org_str) { @@ -234,7 +219,7 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print } // 1 match found, or multiple matches with a common prefix - if (n_found == 1 || match_len > s_len) { + if (q_first == q_last || match_len > s_len) { *compl_str = match_str + s_len; return match_len - s_len; } @@ -245,11 +230,12 @@ size_t mp_repl_autocomplete(const char *str, size_t len, const mp_print_t *print #define MAX_LINE_LEN (4 * WORD_SLOT_LEN) int line_len = MAX_LINE_LEN; // force a newline for first word - for (size_t i = 0; i < dict->map.alloc; i++) { - if (MP_MAP_SLOT_IS_FILLED(&dict->map, i)) { - size_t d_len; - const char *d_str = mp_obj_str_get_data(dict->map.table[i].key, &d_len); - if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + for (qstr q = q_first; q <= q_last; ++q) { + size_t d_len; + const char *d_str = (const char*)qstr_data(q, &d_len); + if (s_len <= d_len && strncmp(s_start, d_str, s_len) == 0) { + mp_load_method_protected(obj, q, dest, true); + if (dest[0] != MP_OBJ_NULL) { int gap = (line_len + WORD_SLOT_LEN - 1) / WORD_SLOT_LEN * WORD_SLOT_LEN - line_len; if (gap < 2) { gap += WORD_SLOT_LEN; diff --git a/python/src/py/runtime.c b/python/src/py/runtime.c index 17e5d235c..8f1063076 100644 --- a/python/src/py/runtime.c +++ b/python/src/py/runtime.c @@ -83,8 +83,10 @@ void mp_init(void) { MICROPY_PORT_INIT_FUNC; #endif + #if MICROPY_ENABLE_COMPILER // optimization disabled by default MP_STATE_VM(mp_optimise_value) = 0; + #endif // init global module dict mp_obj_dict_init(&MP_STATE_VM(mp_loaded_modules_dict), 3); @@ -214,7 +216,7 @@ void mp_delete_global(qstr qst) { } mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { - DEBUG_OP_printf("unary " UINT_FMT " %p\n", op, arg); + DEBUG_OP_printf("unary " UINT_FMT " %q %p\n", op, mp_unary_op_method_name[op], arg); if (op == MP_UNARY_OP_NOT) { // "not x" is the negative of whether "x" is true per Python semantics @@ -275,7 +277,7 @@ mp_obj_t mp_unary_op(mp_unary_op_t op, mp_obj_t arg) { } mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { - DEBUG_OP_printf("binary " UINT_FMT " %p %p\n", op, lhs, rhs); + DEBUG_OP_printf("binary " UINT_FMT " %q %p %p\n", op, mp_binary_op_method_name[op], lhs, rhs); // TODO correctly distinguish inplace operators for mutable objects // lookup logic that CPython uses for +=: @@ -413,7 +415,6 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { // use standard precision return MP_OBJ_NEW_SMALL_INT(lhs_val * rhs_val); } - break; } case MP_BINARY_OP_FLOOR_DIVIDE: case MP_BINARY_OP_INPLACE_FLOOR_DIVIDE: @@ -488,10 +489,10 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { return MP_OBJ_FROM_PTR(tuple); } - case MP_BINARY_OP_LESS: return mp_obj_new_bool(lhs_val < rhs_val); break; - case MP_BINARY_OP_MORE: return mp_obj_new_bool(lhs_val > rhs_val); break; - case MP_BINARY_OP_LESS_EQUAL: return mp_obj_new_bool(lhs_val <= rhs_val); break; - case MP_BINARY_OP_MORE_EQUAL: return mp_obj_new_bool(lhs_val >= rhs_val); break; + case MP_BINARY_OP_LESS: return mp_obj_new_bool(lhs_val < rhs_val); + case MP_BINARY_OP_MORE: return mp_obj_new_bool(lhs_val > rhs_val); + case MP_BINARY_OP_LESS_EQUAL: return mp_obj_new_bool(lhs_val <= rhs_val); + case MP_BINARY_OP_MORE_EQUAL: return mp_obj_new_bool(lhs_val >= rhs_val); default: goto unsupported_op; @@ -523,38 +524,12 @@ mp_obj_t mp_binary_op(mp_binary_op_t op, mp_obj_t lhs, mp_obj_t rhs) { } } - /* deal with `in` - * - * NOTE `a in b` is `b.__contains__(a)`, hence why the generic dispatch - * needs to go below with swapped arguments - */ + // Convert MP_BINARY_OP_IN to MP_BINARY_OP_CONTAINS with swapped args. if (op == MP_BINARY_OP_IN) { - mp_obj_type_t *type = mp_obj_get_type(rhs); - if (type->binary_op != NULL) { - mp_obj_t res = type->binary_op(op, rhs, lhs); - if (res != MP_OBJ_NULL) { - return res; - } - } - if (type->getiter != NULL) { - /* second attempt, walk the iterator */ - mp_obj_iter_buf_t iter_buf; - mp_obj_t iter = mp_getiter(rhs, &iter_buf); - mp_obj_t next; - while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { - if (mp_obj_equal(next, lhs)) { - return mp_const_true; - } - } - return mp_const_false; - } - - if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { - mp_raise_TypeError("object not iterable"); - } else { - nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_TypeError, - "'%s' object is not iterable", mp_obj_get_type_str(rhs))); - } + op = MP_BINARY_OP_CONTAINS; + mp_obj_t temp = lhs; + lhs = rhs; + rhs = temp; } // generic binary_op supplied by type @@ -583,6 +558,20 @@ generic_binary_op: } #endif + if (op == MP_BINARY_OP_CONTAINS) { + // If type didn't support containment then explicitly walk the iterator. + // mp_getiter will raise the appropriate exception if lhs is not iterable. + mp_obj_iter_buf_t iter_buf; + mp_obj_t iter = mp_getiter(lhs, &iter_buf); + mp_obj_t next; + while ((next = mp_iternext(iter)) != MP_OBJ_STOP_ITERATION) { + if (mp_obj_equal(next, rhs)) { + return mp_const_true; + } + } + return mp_const_false; + } + unsupported_op: if (MICROPY_ERROR_REPORTING == MICROPY_ERROR_REPORTING_TERSE) { mp_raise_TypeError("unsupported type for operator"); @@ -681,7 +670,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // allocate memory for the new array of args args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len); - args2 = m_new(mp_obj_t, args2_alloc); + args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self if (self != MP_OBJ_NULL) { @@ -702,7 +691,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // allocate memory for the new array of args args2_alloc = 1 + n_args + len + 2 * (n_kw + kw_dict_len); - args2 = m_new(mp_obj_t, args2_alloc); + args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self if (self != MP_OBJ_NULL) { @@ -718,7 +707,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ // allocate memory for the new array of args args2_alloc = 1 + n_args + 2 * (n_kw + kw_dict_len) + 3; - args2 = m_new(mp_obj_t, args2_alloc); + args2 = mp_nonlocal_alloc(args2_alloc * sizeof(mp_obj_t)); // copy the self if (self != MP_OBJ_NULL) { @@ -735,7 +724,7 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ mp_obj_t item; while ((item = mp_iternext(iterable)) != MP_OBJ_STOP_ITERATION) { if (args2_len >= args2_alloc) { - args2 = m_renew(mp_obj_t, args2, args2_alloc, args2_alloc * 2); + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), args2_alloc * 2 * sizeof(mp_obj_t)); args2_alloc *= 2; } args2[args2_len++] = item; @@ -761,8 +750,8 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ if (MP_MAP_SLOT_IS_FILLED(map, i)) { // the key must be a qstr, so intern it if it's a string mp_obj_t key = map->table[i].key; - if (MP_OBJ_IS_TYPE(key, &mp_type_str)) { - key = mp_obj_str_intern(key); + if (!MP_OBJ_IS_QSTR(key)) { + key = mp_obj_str_intern_checked(key); } args2[args2_len++] = key; args2[args2_len++] = map->table[i].value; @@ -786,13 +775,13 @@ void mp_call_prepare_args_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ if (new_alloc < 4) { new_alloc = 4; } - args2 = m_renew(mp_obj_t, args2, args2_alloc, new_alloc); + args2 = mp_nonlocal_realloc(args2, args2_alloc * sizeof(mp_obj_t), new_alloc * sizeof(mp_obj_t)); args2_alloc = new_alloc; } // the key must be a qstr, so intern it if it's a string - if (MP_OBJ_IS_TYPE(key, &mp_type_str)) { - key = mp_obj_str_intern(key); + if (!MP_OBJ_IS_QSTR(key)) { + key = mp_obj_str_intern_checked(key); } // get the value corresponding to the key @@ -818,7 +807,7 @@ mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_ob mp_call_prepare_args_n_kw_var(have_self, n_args_n_kw, args, &out_args); mp_obj_t res = mp_call_function_n_kw(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args); - m_del(mp_obj_t, out_args.args, out_args.n_alloc); + mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); return res; } @@ -1097,6 +1086,22 @@ void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest) { } } +// Acts like mp_load_method_maybe but catches AttributeError, and all other exceptions if requested +void mp_load_method_protected(mp_obj_t obj, qstr attr, mp_obj_t *dest, bool catch_all_exc) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + mp_load_method_maybe(obj, attr, dest); + nlr_pop(); + } else { + if (!catch_all_exc + && !mp_obj_is_subclass_fast(MP_OBJ_FROM_PTR(((mp_obj_base_t*)nlr.ret_val)->type), + MP_OBJ_FROM_PTR(&mp_type_AttributeError))) { + // Re-raise the exception + nlr_raise(MP_OBJ_FROM_PTR(nlr.ret_val)); + } + } +} + void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t value) { DEBUG_OP_printf("store attr %p.%s <- %p\n", base, qstr_str(attr), value); mp_obj_type_t *type = mp_obj_get_type(base); @@ -1228,13 +1233,12 @@ mp_vm_return_kind_t mp_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t th if (type->iternext != NULL && send_value == mp_const_none) { mp_obj_t ret = type->iternext(self_in); + *ret_val = ret; if (ret != MP_OBJ_STOP_ITERATION) { - *ret_val = ret; return MP_VM_RETURN_YIELD; } else { // Emulate raise StopIteration() // Special case, handled in vm.c - *ret_val = MP_OBJ_NULL; return MP_VM_RETURN_NORMAL; } } @@ -1296,7 +1300,7 @@ mp_vm_return_kind_t mp_resume(mp_obj_t self_in, mp_obj_t send_value, mp_obj_t th // will be propagated up. This behavior is approved by test_pep380.py // test_delegation_of_close_to_non_generator(), // test_delegating_throw_to_non_generator() - *ret_val = throw_value; + *ret_val = mp_make_raise_obj(throw_value); return MP_VM_RETURN_EXCEPTION; } } @@ -1350,6 +1354,8 @@ import_error: return dest[0]; } + #if MICROPY_ENABLE_EXTERNAL_IMPORT + // See if it's a package, then can try FS import if (!mp_obj_is_package(module)) { goto import_error; @@ -1360,11 +1366,12 @@ import_error: const char *pkg_name = mp_obj_str_get_data(dest[0], &pkg_name_len); const uint dot_name_len = pkg_name_len + 1 + qstr_len(name); - char *dot_name = alloca(dot_name_len); + char *dot_name = mp_local_alloc(dot_name_len); memcpy(dot_name, pkg_name, pkg_name_len); dot_name[pkg_name_len] = '.'; memcpy(dot_name + pkg_name_len + 1, qstr_str(name), qstr_len(name)); qstr dot_name_q = qstr_from_strn(dot_name, dot_name_len); + mp_local_free(dot_name); mp_obj_t args[5]; args[0] = MP_OBJ_NEW_QSTR(dot_name_q); @@ -1375,6 +1382,13 @@ import_error: // TODO lookup __import__ and call that instead of going straight to builtin implementation return mp_builtin___import__(5, args); + + #else + + // Package import not supported with external imports disabled + goto import_error; + + #endif } void mp_import_all(mp_obj_t module) { @@ -1468,3 +1482,10 @@ NORETURN void mp_raise_OSError(int errno_) { NORETURN void mp_raise_NotImplementedError(const char *msg) { mp_raise_msg(&mp_type_NotImplementedError, msg); } + +#if MICROPY_STACK_CHECK || MICROPY_ENABLE_PYSTACK +NORETURN void mp_raise_recursion_depth(void) { + nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, + MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded))); +} +#endif diff --git a/python/src/py/runtime.h b/python/src/py/runtime.h index 9c1921cb5..ad65f3f46 100644 --- a/python/src/py/runtime.h +++ b/python/src/py/runtime.h @@ -27,6 +27,7 @@ #define MICROPY_INCLUDED_PY_RUNTIME_H #include "py/mpstate.h" +#include "py/pystack.h" typedef enum { MP_VM_RETURN_NORMAL, @@ -106,8 +107,9 @@ mp_obj_t mp_call_method_n_kw(size_t n_args, size_t n_kw, const mp_obj_t *args); mp_obj_t mp_call_method_n_kw_var(bool have_self, size_t n_args_n_kw, const mp_obj_t *args); mp_obj_t mp_call_method_self_n_kw(mp_obj_t meth, mp_obj_t self, size_t n_args, size_t n_kw, const mp_obj_t *args); // Call function and catch/dump exception - for Python callbacks from C code -void mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg); -void mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2); +// (return MP_OBJ_NULL in case of exception). +mp_obj_t mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg); +mp_obj_t mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2); typedef struct _mp_call_args_t { mp_obj_t fun; @@ -130,6 +132,7 @@ mp_obj_t mp_load_attr(mp_obj_t base, qstr attr); void mp_convert_member_lookup(mp_obj_t obj, const mp_obj_type_t *type, mp_obj_t member, mp_obj_t *dest); void mp_load_method(mp_obj_t base, qstr attr, mp_obj_t *dest); void mp_load_method_maybe(mp_obj_t base, qstr attr, mp_obj_t *dest); +void mp_load_method_protected(mp_obj_t obj, qstr attr, mp_obj_t *dest, bool catch_all_exc); void mp_load_super_method(qstr attr, mp_obj_t *dest); void mp_store_attr(mp_obj_t base, qstr attr, mp_obj_t val); @@ -150,7 +153,7 @@ NORETURN void mp_raise_ValueError(const char *msg); NORETURN void mp_raise_TypeError(const char *msg); NORETURN void mp_raise_NotImplementedError(const char *msg); NORETURN void mp_raise_OSError(int errno_); -NORETURN void mp_exc_recursion_depth(void); +NORETURN void mp_raise_recursion_depth(void); #if MICROPY_BUILTIN_METHOD_CHECK_SELF_ARG #undef mp_check_self diff --git a/python/src/py/runtime0.h b/python/src/py/runtime0.h index a72b7feb7..960532d17 100644 --- a/python/src/py/runtime0.h +++ b/python/src/py/runtime0.h @@ -131,6 +131,10 @@ typedef enum { #endif , + // The runtime will convert MP_BINARY_OP_IN to this operator with swapped args. + // A type should implement this containment operator instead of MP_BINARY_OP_IN. + MP_BINARY_OP_CONTAINS, + MP_BINARY_OP_NUM_RUNTIME, // These 2 are not supported by the runtime and must be synthesised by the emitter diff --git a/python/src/py/runtime_utils.c b/python/src/py/runtime_utils.c index a5c5403ba..b92c6bd76 100644 --- a/python/src/py/runtime_utils.c +++ b/python/src/py/runtime_utils.c @@ -27,22 +27,26 @@ #include "py/runtime.h" -void mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg) { +mp_obj_t mp_call_function_1_protected(mp_obj_t fun, mp_obj_t arg) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_call_function_1(fun, arg); + mp_obj_t ret = mp_call_function_1(fun, arg); nlr_pop(); + return ret; } else { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + return MP_OBJ_NULL; } } -void mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2) { +mp_obj_t mp_call_function_2_protected(mp_obj_t fun, mp_obj_t arg1, mp_obj_t arg2) { nlr_buf_t nlr; if (nlr_push(&nlr) == 0) { - mp_call_function_2(fun, arg1, arg2); + mp_obj_t ret = mp_call_function_2(fun, arg1, arg2); nlr_pop(); + return ret; } else { mp_obj_print_exception(&mp_plat_print, MP_OBJ_FROM_PTR(nlr.ret_val)); + return MP_OBJ_NULL; } } diff --git a/python/src/py/smallint.h b/python/src/py/smallint.h index 42679a78f..6a3c75236 100644 --- a/python/src/py/smallint.h +++ b/python/src/py/smallint.h @@ -50,10 +50,10 @@ #elif MICROPY_OBJ_REPR == MICROPY_OBJ_REPR_D -#define MP_SMALL_INT_MIN ((mp_int_t)(((mp_int_t)0xffffffff80000000) >> 1)) -#define MP_SMALL_INT_FITS(n) ((((n) ^ ((n) << 1)) & 0xffffffff80000000) == 0) +#define MP_SMALL_INT_MIN ((mp_int_t)(((mp_int_t)0xffff800000000000) >> 1)) +#define MP_SMALL_INT_FITS(n) ((((n) ^ ((n) << 1)) & 0xffff800000000000) == 0) // Mask to truncate mp_int_t to positive value -#define MP_SMALL_INT_POSITIVE_MASK ~(0xffffffff80000000 | (0xffffffff80000000 >> 1)) +#define MP_SMALL_INT_POSITIVE_MASK ~(0xffff800000000000 | (0xffff800000000000 >> 1)) #endif diff --git a/python/src/py/stackctrl.c b/python/src/py/stackctrl.c index 7cd35fee2..5c07796bd 100644 --- a/python/src/py/stackctrl.c +++ b/python/src/py/stackctrl.c @@ -48,14 +48,9 @@ void mp_stack_set_limit(mp_uint_t limit) { MP_STATE_THREAD(stack_limit) = limit; } -void mp_exc_recursion_depth(void) { - nlr_raise(mp_obj_new_exception_arg1(&mp_type_RuntimeError, - MP_OBJ_NEW_QSTR(MP_QSTR_maximum_space_recursion_space_depth_space_exceeded))); -} - void mp_stack_check(void) { if (mp_stack_usage() >= MP_STATE_THREAD(stack_limit)) { - mp_exc_recursion_depth(); + mp_raise_recursion_depth(); } } diff --git a/python/src/py/stream.c b/python/src/py/stream.c index 453dee769..393988d2c 100644 --- a/python/src/py/stream.c +++ b/python/src/py/stream.c @@ -32,13 +32,6 @@ #include "py/stream.h" #include "py/runtime.h" -#if MICROPY_STREAMS_NON_BLOCK -#include -#if defined(__MINGW32__) && !defined(__MINGW64_VERSION_MAJOR) -#define EWOULDBLOCK 140 -#endif -#endif - // This file defines generic Python stream read/write methods which // dispatch to the underlying stream interface of an object. @@ -105,13 +98,6 @@ const mp_stream_p_t *mp_get_stream_raise(mp_obj_t self_in, int flags) { return stream_p; } -mp_obj_t mp_stream_close(mp_obj_t stream) { - // TODO: Still consider using ioctl for close - mp_obj_t dest[2]; - mp_load_method(stream, MP_QSTR_close, dest); - return mp_call_method_n_kw(0, 0, dest); -} - STATIC mp_obj_t stream_read_generic(size_t n_args, const mp_obj_t *args, byte flags) { const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_READ); @@ -434,6 +420,17 @@ mp_obj_t mp_stream_unbuffered_iter(mp_obj_t self) { return MP_OBJ_STOP_ITERATION; } +mp_obj_t mp_stream_close(mp_obj_t stream) { + const mp_stream_p_t *stream_p = mp_get_stream_raise(stream, MP_STREAM_OP_IOCTL); + int error; + mp_uint_t res = stream_p->ioctl(stream, MP_STREAM_CLOSE, 0, &error); + if (res == MP_STREAM_ERROR) { + mp_raise_OSError(error); + } + return mp_const_none; +} +MP_DEFINE_CONST_FUN_OBJ_1(mp_stream_close_obj, mp_stream_close); + STATIC mp_obj_t stream_seek(size_t n_args, const mp_obj_t *args) { const mp_stream_p_t *stream_p = mp_get_stream_raise(args[0], MP_STREAM_OP_IOCTL); diff --git a/python/src/py/stream.h b/python/src/py/stream.h index fbe3d7d85..a1d1c4f8a 100644 --- a/python/src/py/stream.h +++ b/python/src/py/stream.h @@ -35,7 +35,7 @@ #define MP_STREAM_FLUSH (1) #define MP_STREAM_SEEK (2) #define MP_STREAM_POLL (3) -//#define MP_STREAM_CLOSE (4) // Not yet implemented +#define MP_STREAM_CLOSE (4) #define MP_STREAM_TIMEOUT (5) // Get/set timeout (single op) #define MP_STREAM_GET_OPTS (6) // Get stream options #define MP_STREAM_SET_OPTS (7) // Set stream options @@ -69,6 +69,7 @@ MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_unbuffered_readline_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_unbuffered_readlines_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_write_obj); MP_DECLARE_CONST_FUN_OBJ_2(mp_stream_write1_obj); +MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_close_obj); MP_DECLARE_CONST_FUN_OBJ_VAR_BETWEEN(mp_stream_seek_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_tell_obj); MP_DECLARE_CONST_FUN_OBJ_1(mp_stream_flush_obj); @@ -106,7 +107,7 @@ int mp_stream_posix_fsync(mp_obj_t stream); #endif #if MICROPY_STREAMS_NON_BLOCK -#define mp_is_nonblocking_error(errno) ((errno) == EAGAIN || (errno) == EWOULDBLOCK) +#define mp_is_nonblocking_error(errno) ((errno) == MP_EAGAIN || (errno) == MP_EWOULDBLOCK) #else #define mp_is_nonblocking_error(errno) (0) #endif diff --git a/python/src/py/unicode.c b/python/src/py/unicode.c index 140b7ba71..935dc9012 100644 --- a/python/src/py/unicode.c +++ b/python/src/py/unicode.c @@ -67,9 +67,9 @@ STATIC const uint8_t attr[] = { AT_LO, AT_LO, AT_LO, AT_PR, AT_PR, AT_PR, AT_PR, 0 }; -// TODO: Rename to str_get_char -unichar utf8_get_char(const byte *s) { #if MICROPY_PY_BUILTINS_STR_UNICODE + +unichar utf8_get_char(const byte *s) { unichar ord = *s++; if (!UTF8_IS_NONASCII(ord)) return ord; ord &= 0x7F; @@ -80,22 +80,14 @@ unichar utf8_get_char(const byte *s) { ord = (ord << 6) | (*s++ & 0x3F); } return ord; -#else - return *s; -#endif } -// TODO: Rename to str_next_char const byte *utf8_next_char(const byte *s) { -#if MICROPY_PY_BUILTINS_STR_UNICODE ++s; while (UTF8_IS_CONT(*s)) { ++s; } return s; -#else - return s + 1; -#endif } mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) { @@ -109,21 +101,18 @@ mp_uint_t utf8_ptr_to_index(const byte *s, const byte *ptr) { return i; } -// TODO: Rename to str_charlen -mp_uint_t unichar_charlen(const char *str, mp_uint_t len) { -#if MICROPY_PY_BUILTINS_STR_UNICODE - mp_uint_t charlen = 0; - for (const char *top = str + len; str < top; ++str) { +size_t utf8_charlen(const byte *str, size_t len) { + size_t charlen = 0; + for (const byte *top = str + len; str < top; ++str) { if (!UTF8_IS_CONT(*str)) { ++charlen; } } return charlen; -#else - return len; -#endif } +#endif + // Be aware: These unichar_is* functions are actually ASCII-only! bool unichar_isspace(unichar c) { return c < 128 && (attr[c] & FL_SPACE) != 0; @@ -183,6 +172,8 @@ mp_uint_t unichar_xdigit_value(unichar c) { return n; } +#if MICROPY_PY_BUILTINS_STR_UNICODE + bool utf8_check(const byte *p, size_t len) { uint8_t need = 0; const byte *end = p + len; @@ -210,3 +201,5 @@ bool utf8_check(const byte *p, size_t len) { } return need == 0; // no pending fragments allowed } + +#endif diff --git a/python/src/py/vm.c b/python/src/py/vm.c index 564200037..8abe65e43 100644 --- a/python/src/py/vm.c +++ b/python/src/py/vm.c @@ -48,14 +48,6 @@ // top element. // Exception stack also grows up, top element is also pointed at. -// Exception stack unwind reasons (WHY_* in CPython-speak) -// TODO perhaps compress this to RETURN=0, JUMP>0, with number of unwinds -// left to do encoded in the JUMP number -typedef enum { - UNWIND_RETURN = 1, - UNWIND_JUMP, -} mp_unwind_reason_t; - #define DECODE_UINT \ mp_uint_t unum = 0; \ do { \ @@ -613,29 +605,18 @@ dispatch_loop: mp_call_method_n_kw(3, 0, sp); SET_TOP(mp_const_none); } else if (MP_OBJ_IS_SMALL_INT(TOP())) { - mp_int_t cause_val = MP_OBJ_SMALL_INT_VALUE(TOP()); - if (cause_val == UNWIND_RETURN) { - // stack: (..., __exit__, ctx_mgr, ret_val, UNWIND_RETURN) - mp_obj_t ret_val = sp[-1]; - sp[-1] = mp_const_none; - sp[0] = mp_const_none; - sp[1] = mp_const_none; - mp_call_method_n_kw(3, 0, sp - 3); - sp[-3] = ret_val; - sp[-2] = MP_OBJ_NEW_SMALL_INT(UNWIND_RETURN); - } else { - assert(cause_val == UNWIND_JUMP); - // stack: (..., __exit__, ctx_mgr, dest_ip, num_exc, UNWIND_JUMP) - mp_obj_t dest_ip = sp[-2]; - mp_obj_t num_exc = sp[-1]; - sp[-2] = mp_const_none; - sp[-1] = mp_const_none; - sp[0] = mp_const_none; - mp_call_method_n_kw(3, 0, sp - 4); - sp[-4] = dest_ip; - sp[-3] = num_exc; - sp[-2] = MP_OBJ_NEW_SMALL_INT(UNWIND_JUMP); - } + // Getting here there are two distinct cases: + // - unwind return, stack: (..., __exit__, ctx_mgr, ret_val, SMALL_INT(-1)) + // - unwind jump, stack: (..., __exit__, ctx_mgr, dest_ip, SMALL_INT(num_exc)) + // For both cases we do exactly the same thing. + mp_obj_t data = sp[-1]; + mp_obj_t cause = sp[0]; + sp[-1] = mp_const_none; + sp[0] = mp_const_none; + sp[1] = mp_const_none; + mp_call_method_n_kw(3, 0, sp - 3); + sp[-3] = data; + sp[-2] = cause; sp -= 2; // we removed (__exit__, ctx_mgr) } else { assert(mp_obj_is_exception_instance(TOP())); @@ -680,10 +661,11 @@ unwind_jump:; // of a "with" block contains the context manager info. // We're going to run "finally" code as a coroutine // (not calling it recursively). Set up a sentinel - // on a stack so it can return back to us when it is + // on the stack so it can return back to us when it is // done (when WITH_CLEANUP or END_FINALLY reached). - PUSH((mp_obj_t)unum); // push number of exception handlers left to unwind - PUSH(MP_OBJ_NEW_SMALL_INT(UNWIND_JUMP)); // push sentinel + // The sentinel is the number of exception handlers left to + // unwind, which is a non-negative integer. + PUSH(MP_OBJ_NEW_SMALL_INT(unum)); ip = exc_sp->handler; // get exception handler byte code address exc_sp--; // pop exception handler goto dispatch_loop; // run the exception handler @@ -720,11 +702,14 @@ unwind_jump:; } else if (MP_OBJ_IS_SMALL_INT(TOP())) { // We finished "finally" coroutine and now dispatch back // to our caller, based on TOS value - mp_unwind_reason_t reason = MP_OBJ_SMALL_INT_VALUE(POP()); - if (reason == UNWIND_RETURN) { + mp_int_t cause = MP_OBJ_SMALL_INT_VALUE(POP()); + if (cause < 0) { + // A negative cause indicates unwind return goto unwind_return; } else { - assert(reason == UNWIND_JUMP); + // Otherwise it's an unwind jump and we must push as a raw + // number the number of exception handlers to unwind + PUSH((mp_obj_t)cause); goto unwind_jump; } } else { @@ -926,18 +911,22 @@ unwind_jump:; code_state->sp = sp; code_state->exc_sp = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block); mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1); - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + deep_recursion_error: + mp_raise_recursion_depth(); + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - deep_recursion_error: - mp_exc_recursion_depth(); - } - #endif } #endif SET_TOP(mp_call_function_n_kw(*sp, unum & 0xff, (unum >> 8) & 0xff, sp + 1)); @@ -963,18 +952,26 @@ unwind_jump:; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args); - m_del(mp_obj_t, out_args.args, out_args.n_alloc); - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + // Freeing args at this point does not follow a LIFO order so only do it if + // pystack is not enabled. For pystack, they are freed when code_state is. + mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); + #endif + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + goto deep_recursion_error; + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - goto deep_recursion_error; - } - #endif } #endif SET_TOP(mp_call_method_n_kw_var(false, unum, sp)); @@ -998,17 +995,21 @@ unwind_jump:; int adjust = (sp[1] == MP_OBJ_NULL) ? 0 : 1; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(*sp, n_args + adjust, n_kw, sp + 2 - adjust); - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + goto deep_recursion_error; + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - goto deep_recursion_error; - } - #endif } #endif SET_TOP(mp_call_method_n_kw(unum & 0xff, (unum >> 8) & 0xff, sp)); @@ -1034,18 +1035,26 @@ unwind_jump:; mp_code_state_t *new_state = mp_obj_fun_bc_prepare_codestate(out_args.fun, out_args.n_args, out_args.n_kw, out_args.args); - m_del(mp_obj_t, out_args.args, out_args.n_alloc); - if (new_state) { + #if !MICROPY_ENABLE_PYSTACK + // Freeing args at this point does not follow a LIFO order so only do it if + // pystack is not enabled. For pystack, they are freed when code_state is. + mp_nonlocal_free(out_args.args, out_args.n_alloc * sizeof(mp_obj_t)); + #endif + #if !MICROPY_ENABLE_PYSTACK + if (new_state == NULL) { + // Couldn't allocate codestate on heap: in the strict case raise + // an exception, otherwise just fall through to stack allocation. + #if MICROPY_STACKLESS_STRICT + goto deep_recursion_error; + #endif + } else + #endif + { new_state->prev = code_state; code_state = new_state; nlr_pop(); goto run_code_state; } - #if MICROPY_STACKLESS_STRICT - else { - goto deep_recursion_error; - } - #endif } #endif SET_TOP(mp_call_method_n_kw_var(true, unum, sp)); @@ -1081,7 +1090,7 @@ unwind_return: // (not calling it recursively). Set up a sentinel // on a stack so it can return back to us when it is // done (when WITH_CLEANUP or END_FINALLY reached). - PUSH(MP_OBJ_NEW_SMALL_INT(UNWIND_RETURN)); + PUSH(MP_OBJ_NEW_SMALL_INT(-1)); ip = exc_sp->handler; exc_sp--; goto dispatch_loop; @@ -1096,7 +1105,15 @@ unwind_return: if (code_state->prev != NULL) { mp_obj_t res = *sp; mp_globals_set(code_state->old_globals); - code_state = code_state->prev; + mp_code_state_t *new_code_state = code_state->prev; + #if MICROPY_ENABLE_PYSTACK + // Free code_state, and args allocated by mp_call_prepare_args_n_kw_var + // (The latter is implicitly freed when using pystack due to its LIFO nature.) + // The sizeof in the following statement does not include the size of the variable + // part of the struct. This arg is anyway not used if pystack is enabled. + mp_nonlocal_free(code_state, sizeof(mp_code_state_t)); + #endif + code_state = new_code_state; *code_state->sp = res; goto run_code_state; } @@ -1105,7 +1122,7 @@ unwind_return: ENTRY(MP_BC_RAISE_VARARGS): { MARK_EXC_IP_SELECTIVE(); - mp_uint_t unum = *ip++; + mp_uint_t unum = *ip; mp_obj_t obj; if (unum == 2) { mp_warning("exception chaining not supported"); @@ -1126,7 +1143,7 @@ unwind_return: RAISE(obj); } } else { - obj = POP(); + obj = TOP(); } obj = mp_make_raise_obj(obj); RAISE(obj); @@ -1149,6 +1166,7 @@ yield: mp_obj_t send_value = POP(); mp_obj_t t_exc = MP_OBJ_NULL; mp_obj_t ret_value; + code_state->sp = sp; // Save sp because it's needed if mp_resume raises StopIteration if (inject_exc != MP_OBJ_NULL) { t_exc = inject_exc; inject_exc = MP_OBJ_NULL; @@ -1164,8 +1182,7 @@ yield: } else if (ret_kind == MP_VM_RETURN_NORMAL) { // Pop exhausted gen sp--; - // TODO: When ret_value can be MP_OBJ_NULL here?? - if (ret_value == MP_OBJ_NULL || ret_value == MP_OBJ_STOP_ITERATION) { + if (ret_value == MP_OBJ_STOP_ITERATION) { // Optimize StopIteration // TODO: get StopIteration's value PUSH(mp_const_none); @@ -1348,7 +1365,8 @@ exception_handler: } else if (*code_state->ip == MP_BC_YIELD_FROM) { // StopIteration inside yield from call means return a value of // yield from, so inject exception's value as yield from's result - *++code_state->sp = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); + // (Instead of stack pop then push we just replace exhausted gen with value) + *code_state->sp = mp_obj_exception_get_value(MP_OBJ_FROM_PTR(nlr.ret_val)); code_state->ip++; // yield from is over, move to next instruction goto outer_dispatch_loop; // continue with dispatch loop } @@ -1361,8 +1379,7 @@ unwind_loop: // set file and line number that the exception occurred at // TODO: don't set traceback for exceptions re-raised by END_FINALLY. // But consider how to handle nested exceptions. - // TODO need a better way of not adding traceback to constant objects (right now, just GeneratorExit_obj and MemoryError_obj) - if (nlr.ret_val != &mp_const_GeneratorExit_obj && nlr.ret_val != &mp_const_MemoryError_obj) { + if (nlr.ret_val != &mp_const_GeneratorExit_obj) { const byte *ip = code_state->fun_bc->bytecode; ip = mp_decode_uint_skip(ip); // skip n_state ip = mp_decode_uint_skip(ip); // skip n_exc_stack @@ -1438,7 +1455,15 @@ unwind_loop: #if MICROPY_STACKLESS } else if (code_state->prev != NULL) { mp_globals_set(code_state->old_globals); - code_state = code_state->prev; + mp_code_state_t *new_code_state = code_state->prev; + #if MICROPY_ENABLE_PYSTACK + // Free code_state, and args allocated by mp_call_prepare_args_n_kw_var + // (The latter is implicitly freed when using pystack due to its LIFO nature.) + // The sizeof in the following statement does not include the size of the variable + // part of the struct. This arg is anyway not used if pystack is enabled. + mp_nonlocal_free(code_state, sizeof(mp_code_state_t)); + #endif + code_state = new_code_state; size_t n_state = mp_decode_uint_value(code_state->fun_bc->bytecode); fastn = &code_state->state[n_state - 1]; exc_stack = (mp_exc_stack_t*)(code_state->state + n_state);