[parsing] Fix constant code point parsing

poincare/src/parsing/parser.cpp

@@ -1,4 +1,5 @@
 #include "parser.h"
+#include <kandinsky/unicode/utf8_decoder.h>
 
 namespace Poincare {
 
@@ -323,7 +324,7 @@ bool Parser::currentTokenIsSpecialIdentifier() const {
 }
 
 void Parser::parseConstant(Expression & leftHandSide, Token::Type stoppingType) {
-  leftHandSide = Constant::Builder(m_currentToken.text()[0]);
+  leftHandSide = Constant::Builder(m_currentToken.codePoint());
   isThereImplicitMultiplication();
 }
 

poincare/src/parsing/token.h

@@ -53,7 +53,7 @@ public:
     Undefined
   };
 
-  Token(Type type) : m_type(type), m_text(0) {};
+  Token(Type type) : m_type(type), m_text(0), m_codePoint(KDCodePointNull) {};
 
   Type type() const { return m_type; }
   bool is(Type t) const { return m_type == t; }
@@ -62,12 +62,14 @@ public:
   Expression expression() const { return m_expression; }
   const char * text() const { return m_text; }
   size_t length() const { return m_length; }
+  CodePoint codePoint() const { return m_codePoint; }
 
   void setExpression(Expression e) { m_expression = e; }
   void setString(const char * text, size_t length) {
     m_text = text;
     m_length = length;
   }
+  void setCodePoint(CodePoint c) { m_codePoint = c; }
   static int CompareNonNullTerminatedName(const char * nonNullTerminatedName, size_t nonNullTerminatedNameLength, const char * nullTerminatedName) {
     /* Compare m_text to name, similarly to strcmp, assuming
      *  - m_text is not null-terminated
@@ -84,6 +86,7 @@ private:
   Expression m_expression;
   const char * m_text;
   size_t m_length;
+  CodePoint m_codePoint;
 };
 
 }

poincare/src/parsing/tokenizer.cpp

@@ -19,7 +19,7 @@ const CodePoint Tokenizer::nextCodePoint(PopTest popTest, CodePoint context, boo
   if (firstCodePoint != KDCodePointNull) {
     CodePoint codePoint = decoder.nextCodePoint();
     while (codePoint.isCombining()) {
-      numberOfBytesForCodePoint = UTF8Decoder::CharSizeOfCodePoint(codePoint);
+      numberOfBytesForCodePoint+= UTF8Decoder::CharSizeOfCodePoint(codePoint);
       codePoint = decoder.nextCodePoint();
     }
   }
@@ -175,12 +175,13 @@ Token Tokenizer::popToken() {
       || c == KDCodePointScriptSmallE)
   {
     Token result(Token::Constant);
-    result.setString(start, 1);
+    result.setCodePoint(c);
     return result;
   }
   if (c == KDCodePointSquareRoot) {
     Token result(Token::Identifier);
-    result.setString(start, 1);
+    // TODO compute size manually?
+    result.setString(start, UTF8Decoder::CharSizeOfCodePoint(KDCodePointSquareRoot));
     return result;
   }
   if (c == KDCodePointEmpty) {