[poincare/matrix] Forbid nested matrices

Scenario: in calculation,
1/matrix(matrix(matrix(matrix(... matrix(1) ... )))) gave kind of a
weird result. These kind of computations also broke the fuzzer.

poincare/include/poincare/expression.h

@@ -112,6 +112,7 @@ class Expression : public TreeHandle {
   friend class IntegralNode;
   template<int T>
   friend class LogarithmNode;
+  friend class MatrixNode;
   friend class NaperianLogarithmNode;
   friend class NAryExpressionNode;
   friend class StoreNode;

poincare/include/poincare/matrix.h

@@ -11,7 +11,7 @@ public:
     m_numberOfRows(0),
     m_numberOfColumns(0) {}
 
-
+  bool hasMatrixChild(Context * context) const;
   int numberOfRows() const { return m_numberOfRows; }
   int numberOfColumns() const { return m_numberOfColumns; }
   virtual void setNumberOfRows(int rows) { assert(rows >= 0); m_numberOfRows = rows; }
@@ -87,7 +87,7 @@ public:
   static constexpr int k_maxNumberOfCoefficients = 100;
 
   // Expression
-  Expression shallowReduce();
+  Expression shallowReduce(Context * context);
 
 private:
   MatrixNode * node() const { return static_cast<MatrixNode *>(Expression::node()); }

poincare/src/expression.cpp

@@ -684,7 +684,6 @@ void Expression::simplifyAndApproximate(Expression * simplifiedExpression, Expre
   /* Case 1: the reduced expression is a matrix: We scan the matrix children to
    * beautify them with the right complex format. */
   if (e.type() == ExpressionNode::Type::Matrix) {
-    // TODO: this method enables to take the complex format into account when the result is a matrix of scalar. It won't work for nested matrices... Find a more elegant and general solution?
     Matrix m = static_cast<Matrix &>(e);
     *simplifiedExpression = Matrix::Builder();
     if (approximateExpression) {
@@ -694,6 +693,7 @@ void Expression::simplifyAndApproximate(Expression * simplifiedExpression, Expre
       Expression simplifiedChild;
       Expression approximateChild = approximateExpression ? Expression() : nullptr;
       e.childAtIndex(i).beautifyAndApproximateScalar(&simplifiedChild, &approximateChild, userReductionContext, context, complexFormat, angleUnit);
+      assert(!simplifiedChild.deepIsMatrix(context));
       static_cast<Matrix *>(simplifiedExpression)->addChildAtIndexInPlace(simplifiedChild, i, i);
       if (approximateExpression) {
         static_cast<Matrix *>(approximateExpression)->addChildAtIndexInPlace(approximateChild, i, i);
@@ -770,7 +770,7 @@ Expression Expression::mapOnMatrixFirstChild(ExpressionNode::ReductionContext re
   }
   matrix.setDimensions(static_cast<Matrix &>(c).numberOfRows(), static_cast<Matrix &>(c).numberOfColumns());
   replaceWithInPlace(matrix);
-  return matrix.shallowReduce();
+  return matrix.shallowReduce(reductionContext.context());
 }
 
 Expression Expression::radianToAngleUnit(Preferences::AngleUnit angleUnit) {

poincare/src/matrix.cpp

@@ -20,6 +20,15 @@ namespace Poincare {
 
 static inline int minInt(int x, int y) { return x < y ? x : y; }
 
+bool MatrixNode::hasMatrixChild(Context * context) const {
+  for (ExpressionNode * c : children()) {
+    if (Expression(c).deepIsMatrix(context)) {
+      return true;
+    }
+  }
+  return false;
+}
+
 void MatrixNode::didAddChildAtIndex(int newNumberOfChildren) {
   setNumberOfRows(1);
   setNumberOfColumns(newNumberOfChildren);
@@ -30,7 +39,7 @@ int MatrixNode::polynomialDegree(Context * context, const char * symbolName) con
 }
 
 Expression MatrixNode::shallowReduce(ReductionContext reductionContext) {
-  return Matrix(this).shallowReduce();
+  return Matrix(this).shallowReduce(reductionContext.context());
 }
 
 Layout MatrixNode::createLayout(Preferences::PrintFloatMode floatDisplayMode, int numberOfSignificantDigits) const {
@@ -390,7 +399,7 @@ Expression Matrix::determinant(ExpressionNode::ReductionContext reductionContext
   return result;
 }
 
-Expression Matrix::shallowReduce() {
+Expression Matrix::shallowReduce(Context * context) {
   {
     Expression e = Expression::defaultShallowReduce();
     e = e.defaultHandleUnitsInChildren();
@@ -398,6 +407,9 @@ Expression Matrix::shallowReduce() {
       return e;
     }
   }
+  if (node()->hasMatrixChild(context)) {
+    return replaceWithUndefinedInPlace();
+  }
   return *this;
 }
 

poincare/src/multiplication.cpp

@@ -677,7 +677,7 @@ Expression Multiplication::privateShallowReduce(ExpressionNode::ReductionContext
       }
     }
     replaceWithInPlace(resultMatrix);
-    return resultMatrix.shallowReduce();
+    return resultMatrix.shallowReduce(reductionContext.context());
   }
 
   /* Step 4: Gather like terms. For example, turn pi^2*pi^3 into pi^5. Thanks to