Fixed some conflicts

python/Makefile

@@ -180,3 +180,7 @@ $(eval $(call rule_for, \
 ))
 
 $(call object_for,$(python_src)): $(BUILD_DIR)/python/port/genhdr/qstrdefs.generated.h
+
+tests_src += $(addprefix python/test/,\
+  mandelbrot.cpp \
+)

python/port/genhdr/qstrdefs.in.h

@@ -193,6 +193,9 @@ Q(<genexpr>)
 Q(<string>)
 Q(<stdin>)
 Q(utf-8)
+#if __EMSCRIPTEN__
+Q(pystack exhausted)
+#endif
 Q(ArithmeticError)
 Q(AssertionError)
 Q(AttributeError)
@@ -249,6 +252,9 @@ Q(__lt__)
 Q(__main__)
 Q(__module__)
 Q(__name__)
+#if __EMSCRIPTEN__
+Q(__ne__)
+#endif
 Q(__new__)
 Q(__next__)
 Q(__path__)
@@ -412,6 +418,10 @@ Q(pop)
 Q(popitem)
 Q(pow)
 Q(print)
+#if __EMSCRIPTEN__
+Q(pystack_space_exhausted)
+Q(pystack_use)
+#endif
 Q(radians)
 Q(randint)
 Q(random)

python/port/helpers.cpp

@@ -1,4 +1,5 @@
 #include "helpers.h"
+#include "port.h"
 #include <ion.h>
 extern "C" {
 #include "mphalport.h"
@@ -18,10 +19,16 @@ bool micropython_port_vm_hook_loop() {
     return false;
   }
 
+  micropython_port_vm_hook_refresh_print();
   // Check if the user asked for an interruption from the keyboard
   return micropython_port_interrupt_if_needed();
 }
 
+void micropython_port_vm_hook_refresh_print() {
+  assert(MicroPython::ExecutionEnvironment::currentExecutionEnvironment() != nullptr);
+  MicroPython::ExecutionEnvironment::currentExecutionEnvironment()->refreshPrintOutput();
+}
+
 bool micropython_port_interruptible_msleep(int32_t delay) {
   assert(delay >= 0);
   /* We don't use millis because the systick drifts when changing the HCLK

python/port/helpers.h

@@ -10,6 +10,7 @@ extern "C" {
 
 // These methods return true if they have been interrupted
 bool micropython_port_vm_hook_loop();
+void micropython_port_vm_hook_refresh_print();
 bool micropython_port_interruptible_msleep(int32_t delay);
 bool micropython_port_interrupt_if_needed();
 int micropython_port_random();

python/port/mod/ion/modion_table.cpp

@@ -8,12 +8,12 @@ extern "C" {
  * use the macros that micropython recommends, and we have to hand build those
  * structs. To avoid errors, we drop in a few static_asserts. */
 
-static_assert(sizeof(mp_fun_1_t) == sizeof(mp_fun_0_t));
-static_assert(sizeof(mp_obj_fun_builtin_fixed_t) == sizeof(mp_obj_base_t) + sizeof(mp_fun_1_t));
+static_assert(sizeof(mp_fun_1_t) == sizeof(mp_fun_0_t), "modion_table badly formed");
+static_assert(sizeof(mp_obj_fun_builtin_fixed_t) == sizeof(mp_obj_base_t) + sizeof(mp_fun_1_t), "modion_keyboard_keydown_obj badly formed");
 
 const mp_obj_fun_builtin_fixed_t modion_keyboard_keydown_obj = {
   {&mp_type_fun_builtin_1},
-  (mp_fun_0_t)modion_keyboard_keydown
+  {(mp_fun_0_t)modion_keyboard_keydown}
 };
 
 STATIC const mp_rom_map_elem_t modion_module_globals_table[] = {
@@ -69,7 +69,7 @@ STATIC const mp_rom_map_elem_t modion_module_globals_table[] = {
 
 STATIC MP_DEFINE_CONST_DICT(modion_module_globals, modion_module_globals_table);
 
-static_assert(sizeof(mp_obj_module_t) == sizeof(mp_obj_base_t) + sizeof(mp_obj_dict_t *));
+static_assert(sizeof(mp_obj_module_t) == sizeof(mp_obj_base_t) + sizeof(mp_obj_dict_t *), "modion_module badly formed");
 
 extern "C" const mp_obj_module_t modion_module = {
   { &mp_type_module },

python/port/mod/kandinsky/modkandinsky.cpp

@@ -52,7 +52,8 @@ mp_obj_t modkandinsky_color(mp_obj_t red, mp_obj_t green, mp_obj_t blue) {
 
 mp_obj_t modkandinsky_get_pixel(mp_obj_t x, mp_obj_t y) {
   KDPoint point(mp_obj_get_int(x), mp_obj_get_int(y));
-  KDColor c = KDIonContext::sharedContext()->getPixel(point);
+  KDColor c;
+  KDIonContext::sharedContext()->getPixel(point, &c);
   return TupleForRGB(c.red(), c.green(), c.blue());
 }
 

python/port/mod/turtle/modturtle.cpp

@@ -3,12 +3,13 @@ extern "C" {
 #include <py/gc.h>
 }
 #include "turtle.h"
+#include "../../port.h"
 
 static Turtle sTurtle;
 
 void modturtle_gc_collect() {
   // Mark the shared sTurtle object as a GC root
-  gc_collect_root((void **)&sTurtle, sizeof(Turtle)/sizeof(void *));
+  MicroPython::collectRootsAtAddress((char *)&sTurtle, sizeof(Turtle));
 }
 
 void modturtle_view_did_disappear() {
@@ -25,6 +26,8 @@ mp_obj_t modturtle___init__() {
 
 mp_obj_t modturtle_reset() {
   sTurtle.reset();
+  // Cf comment on modkandinsky_draw_string
+  micropython_port_interrupt_if_needed();
   return mp_const_none;
 }
 

python/port/mpconfigport.h

@@ -5,6 +5,19 @@
 /* MicroPython configuration options
  * We're not listing the default options as defined in mpconfig.h */
 
+#if __EMSCRIPTEN__
+// Enable a PyStack where most objects are allocated instead of always using the heap
+/* This enables to allocate and free memory in a scope (thus, Python can call
+ * Python) but also has the collateral effect of removing bugs regarding
+ * garbage collection on the web simulator. Indeed, fewer objetcts are
+ * allocated on the heap and the garbage collection is less frequently called.
+ * We suspect that garbage collection failed in javascript because when
+ * collecting roots the transpiled C code is denied access to Javascript
+ * variables that can store pointers to the Python heap. The pointed objects
+ * are therefore erased prematurely. */
+#define MICROPY_ENABLE_PYSTACK (1)
+#endif
+
 // Maximum length of a path in the filesystem
 #define MICROPY_ALLOC_PATH_MAX (32)
 

python/port/port.cpp

@@ -36,6 +36,9 @@ void MicroPython::ExecutionEnvironment::runCode(const char * str) {
   nlr_buf_t nlr;
   if (nlr_push(&nlr) == 0) {
     mp_lexer_t *lex = mp_lexer_new_from_str_len(0, str, strlen(str), false);
+    /* The input type is "single input" because the Python console is supposed
+     * to be fed lines and not files. */
+    // TODO: add a parameter when other input types (file, eval) are required
     mp_parse_tree_t pt = mp_parse(lex, MP_PARSE_SINGLE_INPUT);
     mp_obj_t module_fun = mp_compile(&pt, lex->source_name, MP_EMIT_OPT_NONE, true);
     mp_hal_set_interrupt_char((int)Ion::Keyboard::Key::Back);
@@ -99,13 +102,29 @@ extern "C" {
 }
 
 void MicroPython::init(void * heapStart, void * heapEnd) {
-#if MP_PORT_USE_STACK_SYMBOLS
-  mp_stack_set_top(&_stack_start);
-  mp_stack_set_limit(&_stack_start - &_stack_end);
-#else
+#if __EMSCRIPTEN__
+  static mp_obj_t pystack[1024];
+  mp_pystack_init(pystack, &pystack[MP_ARRAY_SIZE(pystack)]);
+#endif
+
   volatile int stackTop;
-  mp_stack_set_top((void *)(&stackTop));
-  mp_stack_set_limit(8192);
+  void * stackTopAddress = (void *)(&stackTop);
+  /* We delimit the stack part that will be used by Python. The stackTop is the
+   * address of the first object that can be allocated on Python stack. This
+   * boundaries are used:
+   * - by gc_collect to determine where to collect roots of the objects that
+   *   must be kept on the heap
+   * - to check if the maximal recursion depth has been reached. */
+#if MP_PORT_USE_STACK_SYMBOLS
+  mp_stack_set_top(stackTopAddress);
+  size_t stackLimitInBytes = (char *)stackTopAddress - (char *)&_stack_end;
+  mp_stack_set_limit(stackLimitInBytes);
+#else
+  mp_stack_set_top(stackTopAddress);
+  /* The stack limit is set to roughly mimic the maximal recursion depth of the
+   * device - and actually to be slightly less to be sure not to beat the device
+   * performance.  */
+  mp_stack_set_limit(29152);
 #endif
   gc_init(heapStart, heapEnd);
   mp_init();
@@ -119,6 +138,32 @@ void MicroPython::registerScriptProvider(ScriptProvider * s) {
   sScriptProvider = s;
 }
 
+void MicroPython::collectRootsAtAddress(char * address, int byteLength) {
+#if __EMSCRIPTEN__
+   // All objects are aligned, as asserted.
+  assert(((unsigned long)address) % ((unsigned long)sizeof(void *)) == 0);
+  assert(byteLength % sizeof(void *) == 0);
+  gc_collect_root((void **)address, byteLength / sizeof(void *));
+#else
+  for (size_t i = 0; i < sizeof(void *); i++) {
+    /* Objects on the stack are not necessarily aligned on sizeof(void *),
+     * which is also true for pointers refering to the heap. MicroPython
+     * gc_collect_root expects a table of void * that will be scanned every
+     * sizeof(void *) step. So we have to scan the stack repetitively with a
+     * increasing offset to be sure to check every byte for a heap address.
+     * If some memory can be reinterpreted as a pointer in the heap, gc_collect_root
+     * will prevent the destruction of the pointed heap memory. At worst (if
+     * the interpreted pointer was in fact an unaligned object or uninitialized
+     * memory), we will just keep extra objects in the heap which is not optimal
+     * but does not cause any crash. */
+    char * addressWithOffset = address + i;
+    // Ensure to round the length to the ceiling
+    size_t lengthInAddressSize = (byteLength - i + sizeof(void *) - 1)/sizeof(void *);
+    gc_collect_root((void **)addressWithOffset, lengthInAddressSize);
+  }
+#endif
+}
+
 void gc_collect(void) {
   void * python_stack_top = MP_STATE_THREAD(stack_top);
   assert(python_stack_top != NULL);
@@ -131,6 +176,10 @@ void gc_collect(void) {
    * regs is the also the last object on the stack so the stack is bound by
    * &regs and python_stack_top. */
   jmp_buf regs;
+  /* TODO: we use setjmp to get the registers values to look for python heap
+   * root. However, the 'setjmp' does not guarantee that it gets all registers
+   * values. We should check our setjmp implementation for the device and
+   * ensure that it also works for other platforms. */
   setjmp(regs);
 
   void **regs_ptr = (void**)&regs;
@@ -145,7 +194,7 @@ void gc_collect(void) {
     /* To compute the stack length:
      *                                  regs
      *                             <----------->
-     * STACK ->  ...|  |  |  |  |  |--|--|--|--|  |  |  |  |  |  |
+     * STACK <-  ...|  |  |  |  |  |--|--|--|--|  |  |  |  |  |  |
      *                             ^&regs                        ^python_stack_top
      * */
 
@@ -167,8 +216,7 @@ void gc_collect(void) {
   }
   /* Memory error detectors might find an error here as they might split regs
    * and stack memory zones. */
-  gc_collect_root(scanStart, stackLengthInByte/sizeof(void *));
-
+  MicroPython::collectRootsAtAddress((char *)scanStart, stackLengthInByte);
   gc_collect_end();
 }
 

python/port/port.h

@@ -22,6 +22,7 @@ public:
   virtual void hideSandbox() {}
   virtual void resetSandbox() {}
   virtual void printText(const char * text, size_t length) {}
+  virtual void refreshPrintOutput() {}
   void interrupt();
   void setSandboxIsDisplayed(bool display);
 protected:
@@ -33,6 +34,7 @@ private:
 void init(void * heapStart, void * heapEnd);
 void deinit();
 void registerScriptProvider(ScriptProvider * s);
+void collectRootsAtAddress(char * address, int len);
 
 };
 

python/src/py/builtinimport.c

@@ -334,6 +334,10 @@ mp_obj_t mp_builtin___import__(size_t n_args, const mp_obj_t *args) {
         mod_len = new_mod_l;
     }
 
+    if (mod_len == 0) {
+        mp_raise_ValueError(NULL);
+    }
+
     // check if module already exists
     qstr module_name_qstr = mp_obj_str_get_qstr(module_name);
     mp_obj_t module_obj = mp_module_get(module_name_qstr);

python/src/py/qstr.c

@@ -31,6 +31,7 @@
 #include "py/mpstate.h"
 #include "py/qstr.h"
 #include "py/gc.h"
+#include "py/runtime.h"
 
 // NOTE: we are using linear arrays to store and search for qstr's (unique strings, interned strings)
 // ultimately we will replace this with a static hash table of some kind
@@ -192,12 +193,17 @@ qstr qstr_from_str(const char *str) {
 }
 
 qstr qstr_from_strn(const char *str, size_t len) {
-    assert(len < (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN)));
     QSTR_ENTER();
     qstr q = qstr_find_strn(str, len);
     if (q == 0) {
         // qstr does not exist in interned pool so need to add it
 
+        // check that len is not too big
+        if (len >= (1 << (8 * MICROPY_QSTR_BYTES_IN_LEN))) {
+            QSTR_EXIT();
+            mp_raise_msg(&mp_type_RuntimeError, "name too long");
+        }
+
         // compute number of bytes needed to intern this string
         size_t n_bytes = MICROPY_QSTR_BYTES_IN_HASH + MICROPY_QSTR_BYTES_IN_LEN + len + 1;
 

python/test/mandelbrot.cpp

@@ -0,0 +1,69 @@
+#include <quiz.h>
+#include <assert.h>
+#include <string.h>
+#include <python/port/port.h>
+#include <apps/code/app.h>
+
+
+class TestExecutionEnvironment : public MicroPython::ExecutionEnvironment {
+public:
+  void printText(const char * text, size_t length) override;
+};
+
+void TestExecutionEnvironment::printText(const char * text, size_t length) {
+  quiz_print(text);
+}
+
+static constexpr int s_pythonHeapSize = Code::App::k_pythonHeapSize;
+static char s_pythonHeap[s_pythonHeapSize];
+
+static const char * s_mandelbrotScript = R"(#
+def mandelbrot(N_iteration):
+  for x in range(320):
+    for y in range(222):
+      z = complex(0,0)
+      c = complex(3.5*x/319-2.5, -2.5*y/221+1.25)
+      i = 0
+      while (i < N_iteration) and abs(z) < 2:
+        i = i + 1
+        z = z*z+c
+      rgb = int(255*i/N_iteration)
+      col = (int(rgb),int(rgb*0.75),int(rgb*0.25))
+mandelbrot(2)
+print('ok')
+)";
+
+// TODO: this will be obsolete when runCode will take a parameter to choose the input type
+
+void inlineToBeSingleInput(char * buffer, size_t bufferSize, const char * script) {
+  static const char * openExec = "exec(\"";
+  static const char * closeExec = "\")";
+  assert(strlen(script) + strlen(openExec) + strlen(closeExec) < bufferSize);
+  char * bufferChar = buffer;
+  bufferChar += strlcpy(buffer, openExec, bufferSize);
+  const char * scriptChar = script;
+  while (*scriptChar != 0) {
+    assert(bufferChar - buffer + 2 < bufferSize - 1);
+    if (*scriptChar == '\n') {
+      // Turn carriage return in {'\', 'n'} to be processed by exec
+      *bufferChar++ = '\\';
+      *bufferChar++ = 'n';
+    } else {
+      *bufferChar++ = *scriptChar;
+    }
+    scriptChar++;
+  }
+  bufferChar += strlcpy(bufferChar, closeExec, buffer + bufferSize - bufferChar);
+  assert(bufferChar - buffer < bufferSize);
+  *bufferChar = 0;
+}
+
+QUIZ_CASE(python_mandelbrot) {
+  constexpr size_t bufferSize = 500;
+  char buffer[bufferSize];
+  inlineToBeSingleInput(buffer, bufferSize, s_mandelbrotScript);
+  MicroPython::init(s_pythonHeap, s_pythonHeap + s_pythonHeapSize);
+  TestExecutionEnvironment env;
+  env.runCode(buffer);
+  MicroPython::deinit();
+}