From f7892d887fd06ef31fe8e8d896a32e01af2c36e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?L=C3=A9a=20Saviot?= <lea.saviot@numworks.com>
Date: Fri, 31 Jan 2020 11:04:41 +0100
Subject: [PATCH] [poincare/tokenizer] Fix popToken with greek code points

---
 poincare/src/parsing/tokenizer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/poincare/src/parsing/tokenizer.cpp b/poincare/src/parsing/tokenizer.cpp
index b21a26126..c37d62245 100644
--- a/poincare/src/parsing/tokenizer.cpp
+++ b/poincare/src/parsing/tokenizer.cpp
@@ -162,7 +162,7 @@ Token Tokenizer::popToken() {
      * reserved or custom identifier, popIdentifier is called in both cases.
      */
     Token result(Token::Unit);
-    result.setString(start + 1, popIdentifier());
+    result.setString(start + 1, popIdentifier()); // + 1 for the underscore
     return result;
   }
   if (c.isLatinLetter() ||
@@ -170,7 +170,7 @@ Token Tokenizer::popToken() {
       c.isGreekSmallLetter()) // Greek small letter pi is matched earlier
   {
     Token result(Token::Identifier);
-    result.setString(start, 1 + popIdentifier()); // We already popped 1 code point
+    result.setString(start, UTF8Decoder::CharSizeOfCodePoint(c) + popIdentifier()); // We already popped 1 code point
     return result;
   }
   if ('(' <= c && c <= '/') {