UpsilonNumworks/Upsilon
1
0
Fork 0
mirror of https://github.com/UpsilonNumworks/Upsilon.git synced 2026-03-27 17:50:04 +01:00
Code Issues Packages Projects Releases Wiki Activity

[poincare/parser] Syntax error on combining code points

Browse Source
This commit is contained in:
Léa Saviot
2019-02-01 12:07:51 +01:00
committed by Émilie Feral
parent 07b22f44d2
commit a2451aa104
1 changed files with 5 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
poincare/src/parsing/tokenizer.cpp
View File

@@ -15,24 +15,16 @@ static inline bool isDigit(const CodePoint c) {
const CodePoint Tokenizer::nextCodePoint(PopTest popTest, CodePoint context, bool * testResult) {
UTF8Decoder decoder(m_text);
CodePoint firstCodePoint = decoder.nextCodePoint();
CodePoint c = decoder.nextCodePoint();
const char * nextTextPosition = decoder.stringPosition();
if (firstCodePoint != UCodePointNull) {
CodePoint codePoint = decoder.nextCodePoint();
while (codePoint.isCombining()) {
nextTextPosition = decoder.stringPosition();
codePoint = decoder.nextCodePoint();
}
}
// TODO handle combined code points? For now the combining codepoints get dropped.
bool shouldPop = popTest(firstCodePoint, context);
bool shouldPop = popTest(c, context);
if (testResult != nullptr) {
*testResult = shouldPop;
}
if (shouldPop) {
m_text = nextTextPosition;
}
return firstCodePoint;
return c;
}
const CodePoint Tokenizer::popCodePoint() {
@@ -60,6 +52,8 @@ size_t Tokenizer::popWhile(PopTest popTest, CodePoint context) {
}
size_t Tokenizer::popIdentifier() {
/* TODO handle combined code points? For now combining code points will
* trigger a syntax error. */
return popWhile([](CodePoint c, CodePoint context) { return isLetter(c) || isDigit(c) || c == context; }, '_');
}