From f4e887309ded2f8cd9c7bbf31183d0e829345a0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9a=20Saviot?= <lea.saviot@numworks.com>
Date: Thu, 17 Jan 2019 15:35:43 +0100
Subject: [PATCH] [parsing] Fix constant code point parsing

---
 poincare/src/parsing/parser.cpp    | 3 ++-
 poincare/src/parsing/token.h       | 5 ++++-
 poincare/src/parsing/tokenizer.cpp | 7 ++++---
 3 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/poincare/src/parsing/parser.cpp b/poincare/src/parsing/parser.cpp
index 0925f4f4a..4dee65c2f 100644
--- a/poincare/src/parsing/parser.cpp
+++ b/poincare/src/parsing/parser.cpp
@@ -1,4 +1,5 @@
 #include "parser.h"
+#include <kandinsky/unicode/utf8_decoder.h>
 
 namespace Poincare {
 
@@ -323,7 +324,7 @@ bool Parser::currentTokenIsSpecialIdentifier() const {
 }
 
 void Parser::parseConstant(Expression & leftHandSide, Token::Type stoppingType) {
-  leftHandSide = Constant::Builder(m_currentToken.text()[0]);
+  leftHandSide = Constant::Builder(m_currentToken.codePoint());
   isThereImplicitMultiplication();
 }
 
diff --git a/poincare/src/parsing/token.h b/poincare/src/parsing/token.h
index ea1eff7ab..83079df49 100644
--- a/poincare/src/parsing/token.h
+++ b/poincare/src/parsing/token.h
@@ -53,7 +53,7 @@ public:
     Undefined
   };
 
-  Token(Type type) : m_type(type), m_text(0) {};
+  Token(Type type) : m_type(type), m_text(0), m_codePoint(KDCodePointNull) {};
 
   Type type() const { return m_type; }
   bool is(Type t) const { return m_type == t; }
@@ -62,12 +62,14 @@ public:
   Expression expression() const { return m_expression; }
   const char * text() const { return m_text; }
   size_t length() const { return m_length; }
+  CodePoint codePoint() const { return m_codePoint; }
 
   void setExpression(Expression e) { m_expression = e; }
   void setString(const char * text, size_t length) {
     m_text = text;
     m_length = length;
   }
+  void setCodePoint(CodePoint c) { m_codePoint = c; }
   static int CompareNonNullTerminatedName(const char * nonNullTerminatedName, size_t nonNullTerminatedNameLength, const char * nullTerminatedName) {
     /* Compare m_text to name, similarly to strcmp, assuming
      *  - m_text is not null-terminated
@@ -84,6 +86,7 @@ private:
   Expression m_expression;
   const char * m_text;
   size_t m_length;
+  CodePoint m_codePoint;
 };
 
 }
diff --git a/poincare/src/parsing/tokenizer.cpp b/poincare/src/parsing/tokenizer.cpp
index f5143f235..3ec666548 100644
--- a/poincare/src/parsing/tokenizer.cpp
+++ b/poincare/src/parsing/tokenizer.cpp
@@ -19,7 +19,7 @@ const CodePoint Tokenizer::nextCodePoint(PopTest popTest, CodePoint context, boo
   if (firstCodePoint != KDCodePointNull) {
     CodePoint codePoint = decoder.nextCodePoint();
     while (codePoint.isCombining()) {
-      numberOfBytesForCodePoint = UTF8Decoder::CharSizeOfCodePoint(codePoint);
+      numberOfBytesForCodePoint+= UTF8Decoder::CharSizeOfCodePoint(codePoint);
       codePoint = decoder.nextCodePoint();
     }
   }
@@ -175,12 +175,13 @@ Token Tokenizer::popToken() {
       || c == KDCodePointScriptSmallE)
   {
     Token result(Token::Constant);
-    result.setString(start, 1);
+    result.setCodePoint(c);
     return result;
   }
   if (c == KDCodePointSquareRoot) {
     Token result(Token::Identifier);
-    result.setString(start, 1);
+    // TODO compute size manually?
+    result.setString(start, UTF8Decoder::CharSizeOfCodePoint(KDCodePointSquareRoot));
     return result;
   }
   if (c == KDCodePointEmpty) {