[python] test: check wether script execution lead to a Python error

python/port/port.cpp

@@ -30,7 +30,7 @@ MicroPython::ExecutionEnvironment * MicroPython::ExecutionEnvironment::currentEx
   return sCurrentExecutionEnvironment;
 }
 
-void MicroPython::ExecutionEnvironment::runCode(const char * str) {
+bool MicroPython::ExecutionEnvironment::runCode(const char * str) {
   assert(sCurrentExecutionEnvironment == nullptr);
   sCurrentExecutionEnvironment = this;
 
@@ -38,6 +38,7 @@ void MicroPython::ExecutionEnvironment::runCode(const char * str) {
    * for the exception handling (because of print). */
   mp_hal_set_interrupt_char((int)Ion::Keyboard::Key::Back);
 
+  bool runSucceeded = true;
   nlr_buf_t nlr;
   if (nlr_push(&nlr) == 0) {
     mp_lexer_t *lex = mp_lexer_new_from_str_len(0, str, strlen(str), false);
@@ -49,6 +50,7 @@ void MicroPython::ExecutionEnvironment::runCode(const char * str) {
     mp_call_function_0(module_fun);
     nlr_pop();
   } else { // Uncaught exception
+    runSucceeded = false;
     /* mp_obj_print_exception is supposed to handle error printing. However,
      * because we want to print custom information, we copied and modified the
      * content of mp_obj_print_exception instead of calling it. */
@@ -93,6 +95,7 @@ void MicroPython::ExecutionEnvironment::runCode(const char * str) {
 
   assert(sCurrentExecutionEnvironment == this);
   sCurrentExecutionEnvironment = nullptr;
+  return runSucceeded;
 }
 
 void MicroPython::ExecutionEnvironment::interrupt() {

python/port/port.h

@@ -17,7 +17,7 @@ class ExecutionEnvironment {
 public:
   ExecutionEnvironment() {}
   static ExecutionEnvironment * currentExecutionEnvironment();
-  void runCode(const char * );
+  bool runCode(const char * );
   virtual const char * inputText(const char * prompt) { return nullptr; }
 
   // Sandbox

python/test/execution_environment.cpp

@@ -6,7 +6,9 @@
 char TestExecutionEnvironment::s_pythonHeap[TestExecutionEnvironment::s_pythonHeapSize];
 
 void TestExecutionEnvironment::printText(const char * text, size_t length) {
-  quiz_print(text);
+  assert(m_printTextIndex + length < k_maxPrintedTextSize);
+  m_printTextIndex += strlcpy(m_printTextBuffer + m_printTextIndex, text, length + 1);
+  m_printTextBuffer[m_printTextIndex] = 0;
 }
 
 // TODO: this will be obsolete when runCode will take a parameter to choose the input type
@@ -34,12 +36,24 @@ void inlineToBeSingleInput(char * buffer, size_t bufferSize, const char * script
   *bufferChar = 0;
 }
 
-void assert_script_execution_succeeds(const char * script) {
+bool execute_script(const char * script, const char * outputText = nullptr) {
   constexpr size_t bufferSize = 500;
   char buffer[bufferSize];
   inlineToBeSingleInput(buffer, bufferSize, script);
   MicroPython::init(TestExecutionEnvironment::s_pythonHeap, TestExecutionEnvironment::s_pythonHeap + TestExecutionEnvironment::s_pythonHeapSize);
   TestExecutionEnvironment env;
-  env.runCode(buffer);
+  bool executionResult = env.runCode(buffer);
   MicroPython::deinit();
+  if (outputText) {
+    quiz_assert(strcmp(outputText, env.lastPrintedText()) == 0);
+  }
+  return executionResult;
+}
+
+void assert_script_execution_succeeds(const char * script, const char * outputText) {
+  quiz_assert(execute_script(script, outputText));
+}
+
+void assert_script_execution_fails(const char * script) {
+  quiz_assert(!execute_script(script));
 }

python/test/execution_environment.h

@@ -3,13 +3,21 @@
 
 class TestExecutionEnvironment : public MicroPython::ExecutionEnvironment {
 public:
+  TestExecutionEnvironment() : m_printTextIndex(0) {}
   void printText(const char * text, size_t length) override;
+  const char * lastPrintedText() const { return m_printTextBuffer; }
+
   static constexpr int s_pythonHeapSize = Code::App::k_pythonHeapSize;
   static char s_pythonHeap[s_pythonHeapSize];
+private:
+  static constexpr size_t k_maxPrintedTextSize = 256;
+  char m_printTextBuffer[k_maxPrintedTextSize];
+  size_t m_printTextIndex;
 };
 
 
 // TODO: this will be obsolete when runCode will take a parameter to choose the input type
 void inlineToBeSingleInput(char * buffer, size_t bufferSize, const char * script);
 
-void assert_script_execution_succeeds(const char * script);
+void assert_script_execution_succeeds(const char * script, const char * outputText = nullptr);
+void assert_script_execution_fails(const char * script);

python/test/mandelbrot.cpp

@@ -14,7 +14,6 @@ def mandelbrot(N_iteration):
       rgb = int(255*i/N_iteration)
       col = (int(rgb),int(rgb*0.75),int(rgb*0.25))
 mandelbrot(2)
-print('ok')
 )";
 
 QUIZ_CASE(python_mandelbrot) {

python/test/matplotlib.cpp

@@ -5,7 +5,6 @@ static const char * s_pyplotArrowScript = R"(#
 from matplotlib.pyplot import *
 arrow(2,3,4,5)
 show()
-print('ok')
 )";
 
 static const char * s_pyplotAxisScript = R"(#
@@ -15,13 +14,11 @@ axis([2,3,4,5])
 print(axis())
 scatter(0,1)
 show()
-print('ok')
 )";
 
 static const char * s_pyplotAxisErrorScript = R"(#
 from matplotlib.pyplot import *
 axis(2,3,4,5)
-print('fail!')
 )";
 
 static const char * s_pyplotBarScript = R"(#
@@ -32,14 +29,12 @@ bar([],[])
 bar([1,2,3],[1,2,3],2,3)
 bar([1,2,3],[1,2,3],[1,2,3],[1,2,3])
 show()
-print('ok')
 )";
 
 static const char * s_pyplotBarErrorScript = R"(#
 from matplotlib.pyplot import *
 bar([1,2,3],[1,2,3,4],[1,2,3],[1,2,3])
 show()
-print('fail!')
 )";
 
 static const char * s_pyplotGridScript = R"(#
@@ -47,7 +42,6 @@ from matplotlib.pyplot import *
 grid(True)
 grid()
 show()
-print('ok')
 )";
 
 static const char * s_pyplotHistScript = R"(#
@@ -57,7 +51,6 @@ hist([2,3,4,5,6],23)
 hist([2,3,4,5,6],[0,2,3])
 hist([2,3,4,5,6],[0,2,3, 4,5,6,7])
 show()
-print('ok')
 )";
 
 static const char * s_pyplotPlotScript = R"(#
@@ -65,13 +58,11 @@ from matplotlib.pyplot import *
 plot([2,3,4,5,6])
 plot([2,3,4,5,6],[3,4,5,6,7])
 show()
-print('ok')
 )";
 
 static const char * s_pyplotPlotErrorScript = R"(#
 from matplotlib.pyplot import *
 plot([2,3,4,5,6],2)
-print('Fail!')
 )";
 
 static const char * s_pyplotScatterScript = R"(#
@@ -79,33 +70,30 @@ from matplotlib.pyplot import *
 scatter(2,3)
 scatter([2,3,4,5,6],[3,4,5,6,7])
 show()
-print('ok')
 )";
 
 static const char * s_pyplotScatterErrorScript = R"(#
 from matplotlib.pyplot import *
 scatter([2,3,4,5,6],2)
-print('Fail!')
 )";
 
 static const char * s_pyplotTextScript = R"(#
 from matplotlib.pyplot import *
-text(2,3, "hello")
+text(2,3,'hello')
 show()
-print('ok')
 )";
 
 QUIZ_CASE(python_matplotlib_pyplot) {
   assert_script_execution_succeeds(s_pyplotArrowScript);
-  assert_script_execution_succeeds(s_pyplotAxisScript);
-  assert_script_execution_succeeds(s_pyplotAxisErrorScript);
+  assert_script_execution_succeeds(s_pyplotAxisScript, "(2.0, 3.0, 4.0, 5.0)\n");
+  assert_script_execution_fails(s_pyplotAxisErrorScript);
   assert_script_execution_succeeds(s_pyplotBarScript);
-  assert_script_execution_succeeds(s_pyplotBarErrorScript);
+  assert_script_execution_fails(s_pyplotBarErrorScript);
   assert_script_execution_succeeds(s_pyplotGridScript);
   assert_script_execution_succeeds(s_pyplotHistScript);
   assert_script_execution_succeeds(s_pyplotPlotScript);
-  assert_script_execution_succeeds(s_pyplotPlotErrorScript);
+  assert_script_execution_fails(s_pyplotPlotErrorScript);
   assert_script_execution_succeeds(s_pyplotScatterScript);
-  assert_script_execution_succeeds(s_pyplotScatterErrorScript);
+  assert_script_execution_fails(s_pyplotScatterErrorScript);
   assert_script_execution_succeeds(s_pyplotTextScript);
 }