UpsilonNumworks/Upsilon
1
0
Fork 0
mirror of https://github.com/UpsilonNumworks/Upsilon.git synced 2026-01-19 00:37:25 +01:00
Code Issues Packages Projects Releases Wiki Activity

[poincare/tokenizer] Fix popToken with greek code points

Browse Source
This commit is contained in:
Léa Saviot
2020-01-31 11:04:41 +01:00
parent a13ce5df49
commit f7892d887f
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
poincare/src/parsing/tokenizer.cpp
View File

@@ -162,7 +162,7 @@ Token Tokenizer::popToken() {
* reserved or custom identifier, popIdentifier is called in both cases.
*/
Token result(Token::Unit);
result.setString(start + 1, popIdentifier());
result.setString(start + 1, popIdentifier()); // + 1 for the underscore
return result;
}
if (c.isLatinLetter() ||
@@ -170,7 +170,7 @@ Token Tokenizer::popToken() {
c.isGreekSmallLetter()) // Greek small letter pi is matched earlier
{
Token result(Token::Identifier);
result.setString(start, 1 + popIdentifier()); // We already popped 1 code point
result.setString(start, UTF8Decoder::CharSizeOfCodePoint(c) + popIdentifier()); // We already popped 1 code point
return result;
}
if ('(' <= c && c <= '/') {