Rename Codepoint Code point

This commit is contained in:
Léa Saviot
2019-01-11 11:16:14 +01:00
committed by Émilie Feral
parent 42ecd20f12
commit d5df3b74ea
12 changed files with 130 additions and 130 deletions

View File

@@ -1,7 +1,7 @@
// [0x30a].map{|i| "0x" + i.to_s(16) +", // " + [i].pack("U") + " // " + Unicode::Name.of([i].pack("U"))}.join("|")
#include <stdint.h>
uint32_t Codepoints[] = {
uint32_t CodePoints[] = {
0x20, // // SPACE
0x21, // ! // EXCLAMATION MARK
0x22, // " // QUOTATION MARK
@@ -146,4 +146,4 @@ uint32_t Codepoints[] = {
};
int NumberOfCodepoints = sizeof(Codepoints)/sizeof(Codepoints[0]);
int NumberOfCodePoints = sizeof(CodePoints)/sizeof(CodePoints[0]);

View File

@@ -16,7 +16,7 @@
#include FT_FREETYPE_H
#include "unicode_for_symbol.h"
#include "codepoints.h"
#include "code_points.h"
#include "../../ion/src/external/lz4/lz4hc.h"
@@ -94,9 +94,9 @@ int main(int argc, char * argv[]) {
int maxWidth = 0;
int maxAboveBaseline = 0;
int maxBelowBaseline = 0;
for (int i=0; i < NumberOfCodepoints; i++) {
wchar_t codepoint = Codepoints[i];
ENSURE(!FT_Load_Char(face, codepoint, FT_LOAD_RENDER), "Loading character 0x%02x", codepoint);
for (int i=0; i < NumberOfCodePoints; i++) {
wchar_t codePoint = CodePoints[i];
ENSURE(!FT_Load_Char(face, codePoint, FT_LOAD_RENDER), "Loading character 0x%02x", codePoint);
int aboveBaseline = face->glyph->bitmap_top;
int belowBaseline = face->glyph->bitmap.rows - face->glyph->bitmap_top;
int width = face->glyph->bitmap_left + face->glyph->bitmap.width;
@@ -117,7 +117,7 @@ int main(int argc, char * argv[]) {
int grid_size = 1;
int grid_width = 20;
int grid_height = ((NumberOfCodepoints-1)/grid_width)+1;
int grid_height = ((NumberOfCodePoints-1)/grid_width)+1;
bitmap_image.width = grid_width*glyph_width+(grid_width-1)*grid_size;
bitmap_image.height = grid_height*glyph_height+(grid_height-1)*grid_size;
@@ -135,12 +135,12 @@ int main(int argc, char * argv[]) {
}
}
for (int i=0; i<NumberOfCodepoints; i++) {
wchar_t codepoint = Codepoints[i];
for (int i=0; i<NumberOfCodePoints; i++) {
wchar_t codePoint = CodePoints[i];
int x = i%grid_width;
int y = i/grid_width;
// FT_LOAD_RENDER: Render the glyph upon load
ENSURE(!FT_Load_Char(face, codepoint, FT_LOAD_RENDER), "Loading character 0x%08x", codepoint);
ENSURE(!FT_Load_Char(face, codePoint, FT_LOAD_RENDER), "Loading character 0x%08x", codePoint);
//printf("Advances = %dx%d\n", face->glyph->bitmap_left, face->glyph->bitmap_top);
while (face->glyph->bitmap_left < 0) {
// This is a workaround for combining glyphs.
@@ -165,18 +165,18 @@ int main(int argc, char * argv[]) {
fprintf(sourceFile, "/* This file is auto-generated by the rasterizer */\n\n");
fprintf(sourceFile, "#include <kandinsky/font.h>\n\n");
// Step 1 - Build the GlyphIndex <-> UnicodeCodepoint correspondance table
// Step 1 - Build the GlyphIndex <-> UnicodeCodePoint correspondance table
int previousIndex = -1;
uint32_t previousCodepoint = 0;
uint32_t previousCodePoint = 0;
int numberOfPairs = 0;
fprintf(sourceFile, "static constexpr KDFont::CodepointIndexPair table[] = {\n");
for (int i=0; i<NumberOfCodepoints; i++) {
int currentCodepoint = Codepoints[i];
if (currentCodepoint != (previousCodepoint + (i-previousIndex))) {
fprintf(sourceFile, " KDFont::CodepointIndexPair(0x%x, %d),\n", currentCodepoint, i);
previousCodepoint = currentCodepoint;
fprintf(sourceFile, "static constexpr KDFont::CodePointIndexPair table[] = {\n");
for (int i=0; i<NumberOfCodePoints; i++) {
int currentCodePoint = CodePoints[i];
if (currentCodePoint != (previousCodePoint + (i-previousIndex))) {
fprintf(sourceFile, " KDFont::CodePointIndexPair(0x%x, %d),\n", currentCodePoint, i);
previousCodePoint = currentCodePoint;
previousIndex = i;
numberOfPairs++;
}
@@ -195,12 +195,12 @@ int main(int argc, char * argv[]) {
int sizeOfUncompressedGlyphBuffer = glyph_width * glyph_height * greyscaleBitsPerPixel/8;
uint8_t * uncompressedGlyphBuffer = (uint8_t *)malloc(sizeOfUncompressedGlyphBuffer);
uint16_t glyphDataOffset[NumberOfCodepoints+1];
int maxGlyphDataSize = NumberOfCodepoints* sizeOfUncompressedGlyphBuffer;
uint16_t glyphDataOffset[NumberOfCodePoints+1];
int maxGlyphDataSize = NumberOfCodePoints* sizeOfUncompressedGlyphBuffer;
uint8_t * glyphData = (uint8_t *)malloc(maxGlyphDataSize);
uint16_t lastOffset = 0;
for (int character = 0; character < NumberOfCodepoints; character++) {
for (int character = 0; character < NumberOfCodePoints; character++) {
int characterX = (character%grid_width * (glyph_width+grid_size));
int characterY = (character/grid_width * (glyph_height+grid_size));
uint8_t accumulator = 0;
@@ -236,16 +236,16 @@ int main(int argc, char * argv[]) {
glyphDataOffset[character] = lastOffset;
lastOffset += sizeOfCompressedGlyphBuffer;
}
glyphDataOffset[NumberOfCodepoints] = lastOffset;
glyphDataOffset[NumberOfCodePoints] = lastOffset;
fprintf(sourceFile, "static constexpr uint16_t glyphDataOffset[%d] = {", NumberOfCodepoints+1);
prettyPrintArray(sourceFile, 80, 2, glyphDataOffset, NumberOfCodepoints+1);
fprintf(sourceFile, "static constexpr uint16_t glyphDataOffset[%d] = {", NumberOfCodePoints+1);
prettyPrintArray(sourceFile, 80, 2, glyphDataOffset, NumberOfCodePoints+1);
fprintf(sourceFile, "};\n\n");
size_t finalDataSize = lastOffset;
size_t initialDataSize = NumberOfCodepoints * glyph_width * glyph_height;
size_t initialDataSize = NumberOfCodePoints * glyph_width * glyph_height;
fprintf(sourceFile, "/* Rasterized = %5zu bytes (%d glyphs x %d pixels)\n", initialDataSize, NumberOfCodepoints, glyph_width*glyph_height);
fprintf(sourceFile, "/* Rasterized = %5zu bytes (%d glyphs x %d pixels)\n", initialDataSize, NumberOfCodePoints, glyph_width*glyph_height);
fprintf(sourceFile, " * Downsampled = %5lu bytes (1/%d of rasterized)\n", initialDataSize*greyscaleBitsPerPixel/8, 8/greyscaleBitsPerPixel);
fprintf(sourceFile, " * Compressed = %5zu bytes (%.2f%% of rasterized) */\n", finalDataSize, 100.0*finalDataSize/initialDataSize);

View File

@@ -5,7 +5,7 @@
#include <stddef.h>
#include <kandinsky/size.h>
#include <kandinsky/coordinate.h>
#include <kandinsky/unicode/codepoint.h>
#include <kandinsky/unicode/code_point.h>
#include "palette.h"
/* We use UTF-8 encoding. This means that a character is encoded as a code point
@@ -17,11 +17,11 @@
* ASCII characters have the same encoding in ASCII and in UTF-8.
*
* We do not provide a glyph for each of the 1,112,064 valid UTF-8 code points.
* We thus have a table of the glyphs we can draw (uint32_t Codepoints[] in
* kandinsky/fonts/codepoints.h). To easily compute the index of a code point in
* the Codepoints table, we use the m_table matching table: it contains the
* CodepointIndexPairs of the first code point of each series of consecutive
* code points in the Codepoints table. */
* We thus have a table of the glyphs we can draw (uint32_t CodePoints[] in
* kandinsky/fonts/code_points.h). To easily compute the index of a code point in
* the CodePoints table, we use the m_table matching table: it contains the
* CodePointIndexPairs of the first code point of each series of consecutive
* code points in the CodePoints table. */
class KDFont {
private:
@@ -47,19 +47,19 @@ public:
};
using GlyphIndex = uint8_t;
class CodepointIndexPair {
class CodePointIndexPair {
public:
constexpr CodepointIndexPair(Codepoint c, GlyphIndex i) : m_codepoint(c), m_glyphIndex(i) {}
Codepoint codepoint() const { return m_codepoint; }
constexpr CodePointIndexPair(CodePoint c, GlyphIndex i) : m_codePoint(c), m_glyphIndex(i) {}
CodePoint codePoint() const { return m_codePoint; }
GlyphIndex glyphIndex() const { return m_glyphIndex; }
private:
Codepoint m_codepoint;
CodePoint m_codePoint;
GlyphIndex m_glyphIndex;
};
GlyphIndex indexForCodepoint(Codepoint c) const;
GlyphIndex indexForCodePoint(CodePoint c) const;
void setGlyphGreyscalesForCodepoint(Codepoint codepoint, GlyphBuffer * glyphBuffer) const;
void accumulateGlyphGreyscalesForCodepoint(Codepoint codepoint, GlyphBuffer * glyphBuffer) const;
void setGlyphGreyscalesForCodePoint(CodePoint codePoint, GlyphBuffer * glyphBuffer) const;
void accumulateGlyphGreyscalesForCodePoint(CodePoint codePoint, GlyphBuffer * glyphBuffer) const;
using RenderPalette = KDPalette<(1<<k_bitsPerPixel)>;
void colorizeGlyphBuffer(const RenderPalette * renderPalette, GlyphBuffer * glyphBuffer) const;
@@ -69,7 +69,7 @@ public:
}
KDSize glyphSize() const { return m_glyphSize; }
constexpr KDFont(size_t tableLength, const CodepointIndexPair * table, KDCoordinate glyphWidth, KDCoordinate glyphHeight, const uint16_t * glyphDataOffset, const uint8_t * data) :
constexpr KDFont(size_t tableLength, const CodePointIndexPair * table, KDCoordinate glyphWidth, KDCoordinate glyphHeight, const uint16_t * glyphDataOffset, const uint8_t * data) :
m_tableLength(tableLength), m_table(table), m_glyphSize(glyphWidth, glyphHeight), m_glyphDataOffset(glyphDataOffset), m_data(data) { }
private:
void fetchGreyscaleGlyphAtIndex(GlyphIndex index, uint8_t * greyscaleBuffer) const;
@@ -83,7 +83,7 @@ private:
}
size_t m_tableLength;
const CodepointIndexPair * m_table;
const CodePointIndexPair * m_table;
KDSize m_glyphSize;
const uint16_t * m_glyphDataOffset;
const uint8_t * m_data;

View File

@@ -0,0 +1,23 @@
#ifndef KANDINSKY_UNICODE_CODE_POINT_H
#define KANDINSKY_UNICODE_CODE_POINT_H
#include <stdint.h>
class CodePoint {
public:
constexpr CodePoint(uint32_t c) : m_code(c) {}
operator uint16_t() const { return m_code; }
bool isCombining() const {
return (m_code >= 0x300 && m_code <= 0x036F);
}
private:
uint32_t m_code;
};
static constexpr CodePoint Null = 0x0;
static constexpr CodePoint Tabulation = 0x9;
static constexpr CodePoint LineFeed = 0xA;
#endif

View File

@@ -1,23 +0,0 @@
#ifndef KANDINSKY_UNICODE_CODEPOINT_H
#define KANDINSKY_UNICODE_CODEPOINT_H
#include <stdint.h>
class Codepoint {
public:
constexpr Codepoint(uint32_t c) : m_code(c) {}
operator uint16_t() const { return m_code; }
bool isCombining() const {
return (m_code >= 0x300 && m_code <= 0x036F);
}
private:
uint32_t m_code;
};
static constexpr Codepoint Null = 0x0;
static constexpr Codepoint Tabulation = 0x9;
static constexpr Codepoint LineFeed = 0xA;
#endif

View File

@@ -1,12 +1,12 @@
#ifndef KANDINSKY_UNICODE_UTF8DECODER_H
#define KANDINSKY_UNICODE_UTF8DECODER_H
#include "codepoint.h"
#include "code_point.h"
class UTF8Decoder {
public:
UTF8Decoder(const char * string) : m_string(string) {}
Codepoint nextCodepoint();
CodePoint nextCodePoint();
private:
const char * m_string;
};

View File

@@ -13,21 +13,21 @@ KDPoint KDContext::drawString(const char * text, KDPoint p, const KDFont * font,
KDFont::GlyphBuffer glyphBuffer;
UTF8Decoder decoder(text);
Codepoint codepoint = decoder.nextCodepoint();
while (codepoint != Null) {
if (codepoint == LineFeed) {
CodePoint codePoint = decoder.nextCodePoint();
while (codePoint != Null) {
if (codePoint == LineFeed) {
position = KDPoint(0, position.y() + glyphSize.height());
codepoint = decoder.nextCodepoint();
} else if (codepoint == Tabulation) {
codePoint = decoder.nextCodePoint();
} else if (codePoint == Tabulation) {
position = position.translatedBy(KDPoint(k_tabCharacterWidth * glyphSize.width(), 0));
codepoint = decoder.nextCodepoint();
codePoint = decoder.nextCodePoint();
} else {
assert(!codepoint.isCombining());
font->setGlyphGreyscalesForCodepoint(codepoint, &glyphBuffer);
codepoint = decoder.nextCodepoint();
while (codepoint.isCombining()) {
font->accumulateGlyphGreyscalesForCodepoint(codepoint, &glyphBuffer);
codepoint = decoder.nextCodepoint();
assert(!codePoint.isCombining());
font->setGlyphGreyscalesForCodePoint(codePoint, &glyphBuffer);
codePoint = decoder.nextCodePoint();
while (codePoint.isCombining()) {
font->accumulateGlyphGreyscalesForCodePoint(codePoint, &glyphBuffer);
codePoint = decoder.nextCodePoint();
}
font->colorizeGlyphBuffer(&palette, &glyphBuffer);
// Flush accumulated content

View File

@@ -12,31 +12,31 @@ KDSize KDFont::stringSize(const char * text) const {
KDSize stringSize = KDSize(0, m_glyphSize.height());
UTF8Decoder decoder(text);
Codepoint codepoint = decoder.nextCodepoint();
while (codepoint != Null) {
CodePoint codePoint = decoder.nextCodePoint();
while (codePoint != Null) {
KDSize cSize = KDSize(m_glyphSize.width(), 0);
if (codepoint == LineFeed) {
if (codePoint == LineFeed) {
cSize = KDSize(0, m_glyphSize.height());
codepoint = decoder.nextCodepoint();
} else if (codepoint == Tabulation) {
codePoint = decoder.nextCodePoint();
} else if (codePoint == Tabulation) {
cSize = KDSize(k_tabCharacterWidth*m_glyphSize.width(), 0);
} else if (codepoint.isCombining()) {
} else if (codePoint.isCombining()) {
cSize = KDSizeZero;
}
stringSize = KDSize(stringSize.width()+cSize.width(), stringSize.height()+cSize.height());
codepoint = decoder.nextCodepoint();
codePoint = decoder.nextCodePoint();
}
return stringSize;
}
void KDFont::setGlyphGreyscalesForCodepoint(Codepoint codepoint, GlyphBuffer * glyphBuffer) const {
fetchGreyscaleGlyphAtIndex(indexForCodepoint(codepoint), glyphBuffer->greyscaleBuffer());
void KDFont::setGlyphGreyscalesForCodePoint(CodePoint codePoint, GlyphBuffer * glyphBuffer) const {
fetchGreyscaleGlyphAtIndex(indexForCodePoint(codePoint), glyphBuffer->greyscaleBuffer());
}
void KDFont::accumulateGlyphGreyscalesForCodepoint(Codepoint codepoint, GlyphBuffer * glyphBuffer) const {
void KDFont::accumulateGlyphGreyscalesForCodePoint(CodePoint codePoint, GlyphBuffer * glyphBuffer) const {
uint8_t * greyscaleBuffer = glyphBuffer->greyscaleBuffer();
uint8_t * accumulationGreyscaleBuffer = glyphBuffer->secondaryGreyscaleBuffer();
fetchGreyscaleGlyphAtIndex(indexForCodepoint(codepoint), accumulationGreyscaleBuffer);
fetchGreyscaleGlyphAtIndex(indexForCodePoint(codePoint), accumulationGreyscaleBuffer);
for (int i=0; i<m_glyphSize.width()*m_glyphSize.height(); i++) {
greyscaleBuffer[i] |= accumulationGreyscaleBuffer[i];
}
@@ -78,7 +78,7 @@ void KDFont::colorizeGlyphBuffer(const RenderPalette * renderPalette, GlyphBuffe
}
}
KDFont::GlyphIndex KDFont::indexForCodepoint(Codepoint c) const {
KDFont::GlyphIndex KDFont::indexForCodePoint(CodePoint c) const {
#define USE_BINARY_SEARCH 0
#if USE_BINARY_SEARCH
int lowerBound = 0;
@@ -86,12 +86,12 @@ KDFont::GlyphIndex KDFont::indexForCodepoint(Codepoint c) const {
while (true) {
int currentIndex = (lowerBound+upperBound)/2;
// printf("Considering %d in [%d,%d]\n", currentIndex, lowerBound, upperBound);
const CodepointIndexPair * currentPair = m_table + currentIndex;
const CodepointIndexPair * nextPair = (currentIndex + 1) < m_tableLength ? currentPair + 1 : nullptr;
// printf("At this point, currentPair->codepoint() = %d and c = %d\n", currentPair->codepoint(), c);
if (currentPair->codepoint() == c) {
const CodePointIndexPair * currentPair = m_table + currentIndex;
const CodePointIndexPair * nextPair = (currentIndex + 1) < m_tableLength ? currentPair + 1 : nullptr;
// printf("At this point, currentPair->codePoint() = %d and c = %d\n", currentPair->codePoint(), c);
if (currentPair->codePoint() == c) {
return currentPair->glyphIndex();
} else if (currentPair->codepoint() > c) {
} else if (currentPair->codePoint() > c) {
// We need to look below
if (upperBound == currentIndex) {
// There's nothing below. Error out.
@@ -101,9 +101,9 @@ KDFont::GlyphIndex KDFont::indexForCodepoint(Codepoint c) const {
continue;
} else if (nextPair == nullptr) {
return 0;
} else if (nextPair->codepoint() == c) {
} else if (nextPair->codePoint() == c) {
return nextPair->glyphIndex();
} else if (nextPair->codepoint() < c) {
} else if (nextPair->codePoint() < c) {
// We need to look above
if (lowerBound == currentIndex) {
// There's nothing above. Error out.
@@ -113,40 +113,40 @@ KDFont::GlyphIndex KDFont::indexForCodepoint(Codepoint c) const {
continue;
} else {
// At this point,
// currentPair->codepoint < c && nextPair != nullptr && nextPair->codepoint > c
// currentPair->codePoint < c && nextPair != nullptr && nextPair->codePoint > c
// Yay, it's over!
// There can be an empty space between the currentPair and the nextPair
// e.g. currentPair(3,1) and nextPair(9, 4)
// means value at codepoints 3, 4, 5, 6, 7, 8, 9
// means value at codePoints 3, 4, 5, 6, 7, 8, 9
// are glyph identifiers 1, ?, ?, ?, ?, ?, 4
// solved as 1, 2, 3, 0, 0, 0, 4
// Let's hunt down the zeroes
Codepoint lastCodepointOfCurrentPair = currentPair->codepoint() + (nextPair->glyphIndex() - currentPair->glyphIndex() - 1);
if (c > lastCodepointOfCurrentPair) {
CodePoint lastCodePointOfCurrentPair = currentPair->codePoint() + (nextPair->glyphIndex() - currentPair->glyphIndex() - 1);
if (c > lastCodePointOfCurrentPair) {
return 0;
}
return currentPair->glyphIndex() + (c - currentPair->codepoint());
return currentPair->glyphIndex() + (c - currentPair->codePoint());
}
}
#else
const CodepointIndexPair * currentPair = m_table;
if (c < currentPair->codepoint()) {
const CodePointIndexPair * currentPair = m_table;
if (c < currentPair->codePoint()) {
return 0;
}
const CodepointIndexPair * endPair = m_table + m_tableLength - 1;
const CodePointIndexPair * endPair = m_table + m_tableLength - 1;
while (currentPair < endPair) {
const CodepointIndexPair * nextPair = currentPair + 1;
if (c < nextPair->codepoint()) {
Codepoint lastCodepointOfCurrentPair = currentPair->codepoint() + (nextPair->glyphIndex() - currentPair->glyphIndex() - 1);
if (c > lastCodepointOfCurrentPair) {
const CodePointIndexPair * nextPair = currentPair + 1;
if (c < nextPair->codePoint()) {
CodePoint lastCodePointOfCurrentPair = currentPair->codePoint() + (nextPair->glyphIndex() - currentPair->glyphIndex() - 1);
if (c > lastCodePointOfCurrentPair) {
return 0;
}
return currentPair->glyphIndex() + (c - currentPair->codepoint());
return currentPair->glyphIndex() + (c - currentPair->codePoint());
}
currentPair = nextPair;
}
if (endPair->codepoint() == c) {
if (endPair->codePoint() == c) {
return endPair->glyphIndex();
}
return 0;

View File

@@ -15,12 +15,12 @@ static inline uint8_t last_k_bits(uint8_t value, uint8_t bits) {
return (value & ((1<<bits)-1));
}
Codepoint UTF8Decoder::nextCodepoint() {
CodePoint UTF8Decoder::nextCodePoint() {
int leadingOnes = leading_ones(*m_string);
uint32_t result = last_k_bits(*m_string++, 8-leadingOnes-1);
for (int i=0; i<(leadingOnes-1); i++) {
result <<= 6;
result += (*m_string++ & 0x3F);
}
return Codepoint(result);
return CodePoint(result);
}

View File

@@ -2,23 +2,23 @@
#include <kandinsky.h>
#include <assert.h>
static constexpr KDFont::CodepointIndexPair table[] = {
KDFont::CodepointIndexPair(3, 1), // Codepoint, identifier
KDFont::CodepointIndexPair(9, 4),
KDFont::CodepointIndexPair(12, 5),
KDFont::CodepointIndexPair(14, 7)
static constexpr KDFont::CodePointIndexPair table[] = {
KDFont::CodePointIndexPair(3, 1), // CodePoint, identifier
KDFont::CodePointIndexPair(9, 4),
KDFont::CodePointIndexPair(12, 5),
KDFont::CodePointIndexPair(14, 7)
};
constexpr KDFont testFont(4, table, 10, 10, nullptr, nullptr);
const KDFont::GlyphIndex index_for_codepoint[] = {
const KDFont::GlyphIndex index_for_code_point[] = {
/* 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 */
0, 0, 0, 1, 2, 3, 0, 0, 0, 4, 0, 0, 5, 6, 7, 0
};
QUIZ_CASE(kandinsky_font_index_for_codepoint) {
QUIZ_CASE(kandinsky_font_index_for_code_point) {
for (int i=0; i<16; i++) {
KDFont::GlyphIndex result = testFont.indexForCodepoint(i);
quiz_assert(result == index_for_codepoint[i]);
KDFont::GlyphIndex result = testFont.indexForCodePoint(i);
quiz_assert(result == index_for_code_point[i]);
}
}

View File

@@ -1,10 +1,10 @@
#include <quiz.h>
#include <kandinsky/unicode/utf8decoder.h>
void assert_decodes_to(const char * string, Codepoint c) {
void assert_decodes_to(const char * string, CodePoint c) {
UTF8Decoder d(string);
quiz_assert(d.nextCodepoint() == c);
quiz_assert(d.nextCodepoint() == 0);
quiz_assert(d.nextCodePoint() == c);
quiz_assert(d.nextCodePoint() == 0);
}
QUIZ_CASE(kandinsky_utf8_decoder) {

View File

@@ -16,13 +16,13 @@ static inline bool isDigit(const char c) {
const char Tokenizer::nextChar(PopTest popTest, char context, bool * testResult) {
// Beware of chars spaning over more than one byte: use the UTF8Decoder.
UTF8Decoder decoder(m_text);
Codepoint firstCodepoint = decoder.nextCodepoint();
CodePoint firstCodePoint = decoder.nextCodePoint();
int numberOfBytesForChar = 1;
if (firstCodepoint != Null) {
Codepoint codepoint = decoder.nextCodepoint();
while (codepoint.isCombining()) {
if (firstCodePoint != Null) {
CodePoint codePoint = decoder.nextCodePoint();
while (codePoint.isCombining()) {
numberOfBytesForChar++;
codepoint = decoder.nextCodepoint();
codePoint = decoder.nextCodePoint();
}
}
char c = *m_text; // TODO handle combined chars?