diff --git a/ion/Makefile b/ion/Makefile index 1b04baf69..c53ae60ef 100644 --- a/ion/Makefile +++ b/ion/Makefile @@ -31,6 +31,7 @@ ion_src += $(addprefix ion/src/shared/, \ events.cpp \ events_keyboard.cpp \ events_modifier.cpp \ + internal_storage.cpp \ platform_info.cpp \ rtc.cpp \ stack_position.cpp \ diff --git a/ion/include/ion/internal_storage.h b/ion/include/ion/internal_storage.h new file mode 100644 index 000000000..ead06746a --- /dev/null +++ b/ion/include/ion/internal_storage.h @@ -0,0 +1,218 @@ +#ifndef ION_INTERNAL_STORAGE_H +#define ION_INTERNAL_STORAGE_H + +#include +#include +#include "storage.h" + +namespace Ion { + +/* Storage : | Magic | Record1 | Record2 | ... | Magic | + * | Magic | Size1(uint16_t) | FullName1 | Body1 | Size2(uint16_t) | FullName2 | Body2 | ... | Magic | + * + * A record's fullName is baseName.extension. */ + +class StorageDelegate; + +class InternalStorage { +public: + typedef uint16_t record_size_t; + + static constexpr char eqExtension[] = "eq"; + static constexpr char expExtension[] = "exp"; + static constexpr char funcExtension[] = "func"; + static constexpr char seqExtension[] = "seq"; + + constexpr static size_t k_storageSize = 64000; + static_assert(UINT16_MAX >= k_storageSize, "record_size_t not big enough"); + + constexpr static char k_dotChar = '.'; + + class Record { + /* A Record is identified by the CRC32 on its fullName because: + * - A record is identified by its fullName, which is unique + * - We cannot keep the address pointing to the fullName because if another + * record is modified, it might alter our record's fullName address. + * Keeping a buffer with the fullNames will waste memory as we cannot + * forsee the size of the fullNames. */ + friend class Storage; + public: + enum class ErrorStatus { + None = 0, + NameTaken = 1, + NonCompliantName = 2, + NotEnoughSpaceAvailable = 3, + RecordDoesNotExist = 4 + }; + struct Data { + const void * buffer; + size_t size; + }; + Record(const char * fullName = nullptr); + Record(const char * basename, const char * extension); + bool operator==(const Record & other) const { + return m_fullNameCRC32 == other.m_fullNameCRC32; + } + bool operator!=(const Record & other) const { + return !(*this == other); + } +#if ION_STORAGE_LOG + void log(); +#endif + uint32_t checksum(); + bool isNull() const { + return m_fullNameCRC32 == 0; + } + const char * fullName() const; + ErrorStatus setBaseNameWithExtension(const char * baseName, const char * extension); + ErrorStatus setName(const char * fullName); + Data value() const; + ErrorStatus setValue(Data data); + void destroy(); + private: + Record(const char * basename, int basenameLength, const char * extension, int extensionLength); + uint32_t m_fullNameCRC32; + }; + +#if ION_STORAGE_LOG + void log(); +#endif + + size_t availableSize(); + size_t putAvailableSpaceAtEndOfRecord(Record r); + void getAvailableSpaceFromEndOfRecord(Record r, size_t recordAvailableSpace); + uint32_t checksum(); + + // Delegate + void setDelegate(StorageDelegate * delegate) { m_delegate = delegate; } + void notifyChangeToDelegate(const Record r = Record()) const; + Record::ErrorStatus notifyFullnessToDelegate() const; + + int numberOfRecordsWithExtension(const char * extension); + static bool FullNameHasExtension(const char * fullName, const char * extension, size_t extensionLength); + + // Record creation + Record::ErrorStatus createRecordWithFullName(const char * fullName, const void * data, size_t size); + Record::ErrorStatus createRecordWithExtension(const char * baseName, const char * extension, const void * data, size_t size); + + // Record getters + bool hasRecord(Record r) { return pointerOfRecord(r) != nullptr; } + Record recordWithExtensionAtIndex(const char * extension, int index); + Record recordNamed(const char * fullName); + Record recordBaseNamedWithExtension(const char * baseName, const char * extension); + Record recordBaseNamedWithExtensions(const char * baseName, const char * const extension[], size_t numberOfExtensions); + const char * extensionOfRecordBaseNamedWithExtensions(const char * baseName, int baseNameLength, const char * const extension[], size_t numberOfExtensions); + + // Record destruction + void destroyAllRecords(); + void destroyRecordWithBaseNameAndExtension(const char * baseName, const char * extension); + void destroyRecordsWithExtension(const char * extension); + + // Useful + static bool FullNameCompliant(const char * name); + + // Used by Python OS module + int numberOfRecords(); + Record recordAtIndex(int index); +protected: + InternalStorage(); + /* Getters on address in buffer */ + char * pointerOfRecord(const Record record) const; + record_size_t sizeOfRecordStarting(char * start) const; + const char * fullNameOfRecordStarting(char * start) const; + const void * valueOfRecordStarting(char * start) const; + void destroyRecord(const Record record); + + class RecordIterator { + public: + RecordIterator(char * start) : m_recordStart(start) {} + char * operator*() { return m_recordStart; } + RecordIterator& operator++(); + bool operator!=(const RecordIterator& it) const { return m_recordStart != it.m_recordStart; } + private: + char * m_recordStart; + }; + RecordIterator begin() const { + if (sizeOfRecordStarting((char *)m_buffer) == 0) { + return nullptr; + } + return RecordIterator((char *)m_buffer); + }; + RecordIterator end() const { return RecordIterator(nullptr); } + + mutable Record m_lastRecordRetrieved; + mutable char * m_lastRecordRetrievedPointer; +private: + constexpr static uint32_t Magic = 0xEE0BDDBA; + constexpr static size_t k_maxRecordSize = (1 << sizeof(record_size_t)*8); + + /* Getters/Setters on recordID */ + const char * fullNameOfRecord(const Record record); + Record::ErrorStatus setFullNameOfRecord(const Record record, const char * fullName); + Record::ErrorStatus setBaseNameWithExtensionOfRecord(const Record record, const char * baseName, const char * extension); + Record::Data valueOfRecord(const Record record); + Record::ErrorStatus setValueOfRecord(const Record record, Record::Data data); + + /* Overriders */ + size_t overrideSizeAtPosition(char * position, record_size_t size); + size_t overrideFullNameAtPosition(char * position, const char * fullName); + size_t overrideBaseNameWithExtensionAtPosition(char * position, const char * baseName, const char * extension); + size_t overrideValueAtPosition(char * position, const void * data, record_size_t size); + + bool isFullNameTaken(const char * fullName, const Record * recordToExclude = nullptr); + bool isBaseNameWithExtensionTaken(const char * baseName, const char * extension, Record * recordToExclude = nullptr); + bool isNameOfRecordTaken(Record r, const Record * recordToExclude); + char * endBuffer(); + size_t sizeOfBaseNameAndExtension(const char * baseName, const char * extension) const; + size_t sizeOfRecordWithBaseNameAndExtension(const char * baseName, const char * extension, size_t size) const; + size_t sizeOfRecordWithFullName(const char * fullName, size_t size) const; + bool slideBuffer(char * position, int delta); + + Record privateRecordAndExtensionOfRecordBaseNamedWithExtensions(const char * baseName, const char * const extensions[], size_t numberOfExtensions, const char * * extensionResult = nullptr, int baseNameLength = -1); + + uint32_t m_magicHeader; + char m_buffer[k_storageSize]; + uint32_t m_magicFooter; + StorageDelegate * m_delegate; +}; + +/* Some apps memoize records and need to be notified when a record might have + * become invalid. For instance in the Graph app, if f(x) = A+x, and A changed, + * f(x) memoization which stores the reduced expression of f is outdated. + * We could have computed and compared the checksum of the storage to detect + * storage invalidity, but profiling showed that this slows down the execution + * (for example when scrolling the functions list). + * We thus decided to notify a delegate when the storage changes. */ + +class StorageDelegate { +public: + virtual void storageDidChangeForRecord(const InternalStorage::Record record) = 0; + virtual void storageIsFull() = 0; +}; + +// emscripten read and writes must be aligned. +class StorageHelper { +public: + static uint16_t unalignedShort(char * address) { +#if __EMSCRIPTEN__ + uint8_t f1 = *(address); + uint8_t f2 = *(address+1); + uint16_t f = (uint16_t)f1 + (((uint16_t)f2)<<8); + return f; +#else + return *(uint16_t *)address; +#endif + } + static void writeUnalignedShort(uint16_t value, char * address) { +#if __EMSCRIPTEN__ + *((uint8_t *)address) = (uint8_t)(value & ((1 << 8) - 1)); + *((uint8_t *)address+1) = (uint8_t)(value >> 8); +#else + *((uint16_t *)address) = value; +#endif + } +}; + +} + +#endif diff --git a/ion/include/ion/storage.h b/ion/include/ion/storage.h index f1a091d49..5385f4608 100644 --- a/ion/include/ion/storage.h +++ b/ion/include/ion/storage.h @@ -1,239 +1,50 @@ #ifndef ION_STORAGE_H #define ION_STORAGE_H -#include -#include +#include "internal_storage.h" namespace Ion { -/* Storage : | Magic | Record1 | Record2 | ... | Magic | - * | Magic | Size1(uint16_t) | FullName1 | Body1 | Size2(uint16_t) | FullName2 | Body2 | ... | Magic | - * - * A record's fullName is baseName.extension. */ - -class StorageDelegate; - -class Storage { +class Storage : public InternalStorage { public: - typedef uint16_t record_size_t; + using InternalStorage::Record; - constexpr static size_t k_storageSize = 64000; - static_assert(UINT16_MAX >= k_storageSize, "record_size_t not big enough"); + using InternalStorage::expExtension; + using InternalStorage::funcExtension; + using InternalStorage::seqExtension; + using InternalStorage::eqExtension; static Storage * sharedStorage(); - constexpr static char k_dotChar = '.'; - - static constexpr char eqExtension[] = "eq"; - static constexpr char expExtension[] = "exp"; - static constexpr char funcExtension[] = "func"; - static constexpr char seqExtension[] = "seq"; - - class Record { - /* A Record is identified by the CRC32 on its fullName because: - * - A record is identified by its fullName, which is unique - * - We cannot keep the address pointing to the fullName because if another - * record is modified, it might alter our record's fullName address. - * Keeping a buffer with the fullNames will waste memory as we cannot - * forsee the size of the fullNames. */ - friend class Storage; - public: - enum class ErrorStatus { - None = 0, - NameTaken = 1, - NonCompliantName = 2, - NotEnoughSpaceAvailable = 3, - RecordDoesNotExist = 4 - }; - struct Data { - const void * buffer; - size_t size; - }; - Record(const char * fullName = nullptr); - Record(const char * basename, const char * extension); - bool operator==(const Record & other) const { - return m_fullNameCRC32 == other.m_fullNameCRC32; - } - bool operator!=(const Record & other) const { - return !(*this == other); - } -#if ION_STORAGE_LOG - void log(); -#endif - uint32_t checksum(); - bool isNull() const { - return m_fullNameCRC32 == 0; - } - const char * fullName() const { - return Storage::sharedStorage()->fullNameOfRecord(*this); - } - ErrorStatus setBaseNameWithExtension(const char * baseName, const char * extension) { - return Storage::sharedStorage()->setBaseNameWithExtensionOfRecord(*this, baseName, extension); - } - ErrorStatus setName(const char * fullName) { - return Storage::sharedStorage()->setFullNameOfRecord(*this, fullName); - } - Data value() const { - return Storage::sharedStorage()->valueOfRecord(*this); - } - ErrorStatus setValue(Data data) { - return Storage::sharedStorage()->setValueOfRecord(*this, data); - } - void destroy() { - return Storage::sharedStorage()->destroyRecord(*this); - } - private: - Record(const char * basename, int basenameLength, const char * extension, int extensionLength); - uint32_t m_fullNameCRC32; - }; - -#if ION_STORAGE_LOG - void log(); -#endif size_t availableSize(); size_t putAvailableSpaceAtEndOfRecord(Record r); void getAvailableSpaceFromEndOfRecord(Record r, size_t recordAvailableSpace); - uint32_t checksum(); - - // Delegate - void setDelegate(StorageDelegate * delegate) { m_delegate = delegate; } - void notifyChangeToDelegate(const Record r = Record()) const; - Record::ErrorStatus notifyFullnessToDelegate() const; int numberOfRecordsWithExtension(const char * extension); - static bool FullNameHasExtension(const char * fullName, const char * extension, size_t extensionLength); - // Record creation Record::ErrorStatus createRecordWithFullName(const char * fullName, const void * data, size_t size); Record::ErrorStatus createRecordWithExtension(const char * baseName, const char * extension, const void * data, size_t size); - // Record getters - bool hasRecord(Record r) { return pointerOfRecord(r) != nullptr; } + bool hasRecord(Record r); + Record recordWithExtensionAtIndex(const char * extension, int index); Record recordNamed(const char * fullName); Record recordBaseNamedWithExtension(const char * baseName, const char * extension); Record recordBaseNamedWithExtensions(const char * baseName, const char * const extension[], size_t numberOfExtensions); const char * extensionOfRecordBaseNamedWithExtensions(const char * baseName, int baseNameLength, const char * const extension[], size_t numberOfExtensions); - // Record destruction - void destroyAllRecords(); - void destroyRecordWithBaseNameAndExtension(const char * baseName, const char * extension); - void destroyRecordsWithExtension(const char * extension); - - // Useful - static bool FullNameCompliant(const char * name); - - // Used by Python OS module int numberOfRecords(); - Record recordAtIndex(int index); // Unlike realRecordAtIndex, this ignore trash + Record recordAtIndex(int index); + void destroyRecord(Record record); - // Trash void reinsertTrash(const char * extension); + void emptyTrash(); private: - constexpr static uint32_t Magic = 0xEE0BDDBA; - constexpr static size_t k_maxRecordSize = (1 << sizeof(record_size_t)*8); + Storage(): + InternalStorage() {} - Storage(); - - /* Getters/Setters on recordID */ - const char * fullNameOfRecord(const Record record); - Record::ErrorStatus setFullNameOfRecord(const Record record, const char * fullName); - Record::ErrorStatus setBaseNameWithExtensionOfRecord(const Record record, const char * baseName, const char * extension); - Record::Data valueOfRecord(const Record record); - Record::ErrorStatus setValueOfRecord(const Record record, Record::Data data); - void destroyRecord(const Record record); - void realDestroyRecord(const Record record); - - /* Getters on address in buffer */ - char * pointerOfRecord(const Record record) const; - record_size_t sizeOfRecordStarting(char * start) const; - const char * fullNameOfRecordStarting(char * start) const; - const void * valueOfRecordStarting(char * start) const; - - /* Trash */ - void emptyTrash(); - Storage::Record realRecordAtIndex(int index); - int realNumberOfRecords(); - - /* Overriders */ - size_t overrideSizeAtPosition(char * position, record_size_t size); - size_t overrideFullNameAtPosition(char * position, const char * fullName); - size_t overrideBaseNameWithExtensionAtPosition(char * position, const char * baseName, const char * extension); - size_t overrideValueAtPosition(char * position, const void * data, record_size_t size); - - size_t realAvailableSize(); - bool isFullNameTaken(const char * fullName, const Record * recordToExclude = nullptr); - bool isBaseNameWithExtensionTaken(const char * baseName, const char * extension, Record * recordToExclude = nullptr); - bool isNameOfRecordTaken(Record r, const Record * recordToExclude); - char * endBuffer(); - size_t sizeOfBaseNameAndExtension(const char * baseName, const char * extension) const; - size_t sizeOfRecordWithBaseNameAndExtension(const char * baseName, const char * extension, size_t size) const; - size_t sizeOfRecordWithFullName(const char * fullName, size_t size) const; - bool slideBuffer(char * position, int delta); - class RecordIterator { - public: - RecordIterator(char * start) : m_recordStart(start) {} - char * operator*() { return m_recordStart; } - RecordIterator& operator++(); - bool operator!=(const RecordIterator& it) const { return m_recordStart != it.m_recordStart; } - private: - char * m_recordStart; - }; - RecordIterator begin() const { - if (sizeOfRecordStarting((char *)m_buffer) == 0) { - return nullptr; - } - return RecordIterator((char *)m_buffer); - }; - RecordIterator end() const { return RecordIterator(nullptr); } - - Record privateRecordAndExtensionOfRecordBaseNamedWithExtensions(const char * baseName, const char * const extensions[], size_t numberOfExtensions, const char * * extensionResult = nullptr, int baseNameLength = -1); - - uint32_t m_magicHeader; - char m_buffer[k_storageSize]; Record m_trashRecord; - uint32_t m_magicFooter; - StorageDelegate * m_delegate; - mutable Record m_lastRecordRetrieved; - mutable char * m_lastRecordRetrievedPointer; -}; - -/* Some apps memoize records and need to be notified when a record might have - * become invalid. For instance in the Graph app, if f(x) = A+x, and A changed, - * f(x) memoization which stores the reduced expression of f is outdated. - * We could have computed and compared the checksum of the storage to detect - * storage invalidity, but profiling showed that this slows down the execution - * (for example when scrolling the functions list). - * We thus decided to notify a delegate when the storage changes. */ - -class StorageDelegate { -public: - virtual void storageDidChangeForRecord(const Storage::Record record) = 0; - virtual void storageIsFull() = 0; -}; - -// emscripten read and writes must be aligned. -class StorageHelper { -public: - static uint16_t unalignedShort(char * address) { -#if __EMSCRIPTEN__ - uint8_t f1 = *(address); - uint8_t f2 = *(address+1); - uint16_t f = (uint16_t)f1 + (((uint16_t)f2)<<8); - return f; -#else - return *(uint16_t *)address; -#endif - } - static void writeUnalignedShort(uint16_t value, char * address) { -#if __EMSCRIPTEN__ - *((uint8_t *)address) = (uint8_t)(value & ((1 << 8) - 1)); - *((uint8_t *)address+1) = (uint8_t)(value >> 8); -#else - *((uint16_t *)address) = value; -#endif - } }; } diff --git a/ion/src/shared/internal_storage.cpp b/ion/src/shared/internal_storage.cpp new file mode 100644 index 000000000..5f12eb460 --- /dev/null +++ b/ion/src/shared/internal_storage.cpp @@ -0,0 +1,647 @@ +#include +#include +#include +#include +#include "storage.h" +#if ION_STORAGE_LOG +#include +#endif + +namespace Ion { + +/* We want to implement a simple singleton pattern, to make sure the storage is + * initialized on first use, therefore preventing the static init order fiasco. + * That being said, we rely on knowing where the storage resides in the device's + * memory at compile time. Indeed, we want to advertise the static storage's + * memory address in the PlatformInfo structure (so that we can read and write + * it in DFU). + * Using a "static Storage storage;" variable makes it a local symbol at best, + * preventing the PlatformInfo from retrieving its address. And making the + * Storage variable global yields the static init fiasco issue. We're working + * around both issues by creating a global staticStorageArea buffer, and by + * placement-newing the Storage into that area on first use. */ + +/* The InternalStorage is handle all records. Then the "normal" Storage handle + * a trash */ + +constexpr char InternalStorage::expExtension[]; +constexpr char InternalStorage::funcExtension[]; +constexpr char InternalStorage::seqExtension[]; +constexpr char InternalStorage::eqExtension[]; + +// RECORD + +const char * InternalStorage::Record::fullName() const { + return Storage::sharedStorage()->fullNameOfRecord(*this); +} +InternalStorage::Record::ErrorStatus InternalStorage::Record::setBaseNameWithExtension(const char * baseName, const char * extension) { + return Storage::sharedStorage()->setBaseNameWithExtensionOfRecord(*this, baseName, extension); +} +InternalStorage::Record::ErrorStatus InternalStorage::Record::setName(const char * fullName) { + return Storage::sharedStorage()->setFullNameOfRecord(*this, fullName); +} +InternalStorage::Record::Data InternalStorage::Record::value() const { + return Storage::sharedStorage()->valueOfRecord(*this); +} +InternalStorage::Record::ErrorStatus InternalStorage::Record::setValue(Data data) { + return Storage::sharedStorage()->setValueOfRecord(*this, data); +} +void InternalStorage::Record::destroy() { + return Storage::sharedStorage()->destroyRecord(*this); +} + +InternalStorage::Record::Record(const char * fullName) { + if (fullName == nullptr) { + m_fullNameCRC32 = 0; + return; + } + const char * dotChar = UTF8Helper::CodePointSearch(fullName, k_dotChar); + // If no extension, return empty record + if (*dotChar == 0 || *(dotChar+1) == 0) { + m_fullNameCRC32 = 0; + return; + } + new (this) Record(fullName, dotChar - fullName, dotChar+1, (fullName + strlen(fullName)) - (dotChar+1)); +} + +InternalStorage::Record::Record(const char * baseName, const char * extension) { + if (baseName == nullptr) { + assert(extension == nullptr); + m_fullNameCRC32 = 0; + return; + } + new (this) Record(baseName, strlen(baseName), extension, strlen(extension)); +} + + +#if ION_STORAGE_LOG + +void InternalStorage::Record::log() { + std::cout << "Name: " << fullName() << std::endl; + std::cout << " Value (" << value().size << "): " << (char *)value().buffer << "\n\n" << std::endl; +} +#endif + +uint32_t InternalStorage::Record::checksum() { + uint32_t crc32Results[2]; + crc32Results[0] = m_fullNameCRC32; + Data data = value(); + crc32Results[1] = Ion::crc32Byte((const uint8_t *)data.buffer, data.size); + return Ion::crc32Word(crc32Results, 2); +} + +InternalStorage::Record::Record(const char * basename, int basenameLength, const char * extension, int extensionLength) { + assert(basename != nullptr); + assert(extension != nullptr); + + // We compute the CRC32 of the CRC32s of the basename and the extension + uint32_t crc32Results[2]; + crc32Results[0] = Ion::crc32Byte((const uint8_t *)basename, basenameLength); + crc32Results[1] = Ion::crc32Byte((const uint8_t *)extension, extensionLength); + m_fullNameCRC32 = Ion::crc32Word(crc32Results, 2); +} + +// STORAGE + +#if ION_STORAGE_LOG +void InternalStorage::log() { + for (char * p : *this) { + const char * currentName = fullNameOfRecordStarting(p); + Record(currentName).log(); + } +} +#endif + +size_t InternalStorage::availableSize() { + /* TODO maybe do: availableSize(char ** endBuffer) to get the endBuffer if it + * is needed after calling availableSize */ + assert(k_storageSize >= (endBuffer() - m_buffer) + sizeof(record_size_t)); + return k_storageSize-(endBuffer()-m_buffer)-sizeof(record_size_t); +} + +size_t InternalStorage::putAvailableSpaceAtEndOfRecord(InternalStorage::Record r) { + char * p = pointerOfRecord(r); + size_t previousRecordSize = sizeOfRecordStarting(p); + size_t availableStorageSize = availableSize(); + char * nextRecord = p + previousRecordSize; + memmove(nextRecord + availableStorageSize, + nextRecord, + (m_buffer + k_storageSize - availableStorageSize) - nextRecord); + size_t newRecordSize = previousRecordSize + availableStorageSize; + overrideSizeAtPosition(p, (record_size_t)newRecordSize); + return newRecordSize; +} + +void InternalStorage::getAvailableSpaceFromEndOfRecord(Record r, size_t recordAvailableSpace) { + char * p = pointerOfRecord(r); + size_t previousRecordSize = sizeOfRecordStarting(p); + char * nextRecord = p + previousRecordSize; + memmove(nextRecord - recordAvailableSpace, + nextRecord, + m_buffer + k_storageSize - nextRecord); + overrideSizeAtPosition(p, (record_size_t)(previousRecordSize - recordAvailableSpace)); +} + +uint32_t InternalStorage::checksum() { + return Ion::crc32Byte((const uint8_t *) m_buffer, endBuffer()-m_buffer); +} + +void InternalStorage::notifyChangeToDelegate(const Record record) const { + m_lastRecordRetrieved = Record(nullptr); + m_lastRecordRetrievedPointer = nullptr; + if (m_delegate != nullptr) { + m_delegate->storageDidChangeForRecord(record); + } +} + +InternalStorage::Record::ErrorStatus InternalStorage::notifyFullnessToDelegate() const { + if (m_delegate != nullptr) { + m_delegate->storageIsFull(); + } + return Record::ErrorStatus::NotEnoughSpaceAvailable; +} + +InternalStorage::Record::ErrorStatus InternalStorage::createRecordWithFullName(const char * fullName, const void * data, size_t size) { + size_t recordSize = sizeOfRecordWithFullName(fullName, size); + if (recordSize >= k_maxRecordSize || recordSize > availableSize()) { + return notifyFullnessToDelegate(); + } + if (isFullNameTaken(fullName)) { + return Record::ErrorStatus::NameTaken; + } + // Find the end of data + char * newRecordAddress = endBuffer(); + char * newRecord = newRecordAddress; + // Fill totalSize + newRecord += overrideSizeAtPosition(newRecord, (record_size_t)recordSize); + // Fill name + newRecord += overrideFullNameAtPosition(newRecord, fullName); + // Fill data + newRecord += overrideValueAtPosition(newRecord, data, size); + // Next Record is null-sized + overrideSizeAtPosition(newRecord, 0); + Record r = Record(fullName); + notifyChangeToDelegate(r); + m_lastRecordRetrieved = r; + m_lastRecordRetrievedPointer = newRecordAddress; + return Record::ErrorStatus::None; +} + +InternalStorage::Record::ErrorStatus InternalStorage::createRecordWithExtension(const char * baseName, const char * extension, const void * data, size_t size) { + size_t recordSize = sizeOfRecordWithBaseNameAndExtension(baseName, extension, size); + if (recordSize >= k_maxRecordSize || recordSize > availableSize()) { + return notifyFullnessToDelegate(); + } + if (isBaseNameWithExtensionTaken(baseName, extension)) { + return Record::ErrorStatus::NameTaken; + } + // Find the end of data + char * newRecordAddress = endBuffer(); + char * newRecord = newRecordAddress; + // Fill totalSize + newRecord += overrideSizeAtPosition(newRecord, (record_size_t)recordSize); + // Fill name + newRecord += overrideBaseNameWithExtensionAtPosition(newRecord, baseName, extension); + // Fill data + newRecord += overrideValueAtPosition(newRecord, data, size); + // Next Record is null-sized + overrideSizeAtPosition(newRecord, 0); + Record r = Record(fullNameOfRecordStarting(newRecordAddress)); + notifyChangeToDelegate(r); + m_lastRecordRetrieved = r; + m_lastRecordRetrievedPointer = newRecordAddress; + return Record::ErrorStatus::None; +} + +int InternalStorage::numberOfRecordsWithExtension(const char * extension) { + int count = 0; + size_t extensionLength = strlen(extension); + for (char * p : *this) { + const char * name = fullNameOfRecordStarting(p); + if (FullNameHasExtension(name, extension, extensionLength)) { + count++; + } + } + return count; +} + +int InternalStorage::numberOfRecords() { + int count = 0; + for (char * p : *this) { + const char * name = fullNameOfRecordStarting(p); + count++; + } + return count; +} + +InternalStorage::Record InternalStorage::recordAtIndex(int index) { + int currentIndex = -1; + const char * name = nullptr; + char * recordAddress = nullptr; + for (char * p : *this) { + const char * currentName = fullNameOfRecordStarting(p); + currentIndex++; + if (currentIndex == index) { + recordAddress = p; + name = currentName; + break; + } + } + if (name == nullptr) { + return Record(); + } + Record r = Record(name); + m_lastRecordRetrieved = r; + m_lastRecordRetrievedPointer = recordAddress; + return Record(name); +} + +InternalStorage::Record InternalStorage::recordWithExtensionAtIndex(const char * extension, int index) { + int currentIndex = -1; + const char * name = nullptr; + size_t extensionLength = strlen(extension); + char * recordAddress = nullptr; + for (char * p : *this) { + const char * currentName = fullNameOfRecordStarting(p); + if (FullNameHasExtension(currentName, extension, extensionLength)) { + currentIndex++; + } + if (currentIndex == index) { + recordAddress = p; + name = currentName; + break; + } + } + if (name == nullptr) { + return Record(); + } + Record r = Record(name); + m_lastRecordRetrieved = r; + m_lastRecordRetrievedPointer = recordAddress; + return Record(name); +} + +InternalStorage::Record InternalStorage::recordNamed(const char * fullName) { + if (fullName == nullptr) { + return Record(); + } + Record r = Record(fullName); + char * p = pointerOfRecord(r); + if (p != nullptr) { + return r; + } + return Record(); +} + +InternalStorage::Record InternalStorage::recordBaseNamedWithExtension(const char * baseName, const char * extension) { + const char * extensions[1] = {extension}; + return recordBaseNamedWithExtensions(baseName, extensions, 1); +} + +InternalStorage::Record InternalStorage::recordBaseNamedWithExtensions(const char * baseName, const char * const extensions[], size_t numberOfExtensions) { + return privateRecordAndExtensionOfRecordBaseNamedWithExtensions(baseName, extensions, numberOfExtensions); +} + +const char * InternalStorage::extensionOfRecordBaseNamedWithExtensions(const char * baseName, int baseNameLength, const char * const extensions[], size_t numberOfExtensions) { + const char * result = nullptr; + privateRecordAndExtensionOfRecordBaseNamedWithExtensions(baseName, extensions, numberOfExtensions, &result, baseNameLength); + return result; +} + +void InternalStorage::destroyAllRecords() { + overrideSizeAtPosition(m_buffer, 0); + notifyChangeToDelegate(); +} + +void InternalStorage::destroyRecordWithBaseNameAndExtension(const char * baseName, const char * extension) { + recordBaseNamedWithExtension(baseName, extension).destroy(); +} + +void InternalStorage::destroyRecordsWithExtension(const char * extension) { + size_t extensionLength = strlen(extension); + char * currentRecordStart = (char *)m_buffer; + bool didChange = false; + while (currentRecordStart != nullptr && sizeOfRecordStarting(currentRecordStart) != 0) { + const char * currentFullName = fullNameOfRecordStarting(currentRecordStart); + if (FullNameHasExtension(currentFullName, extension, extensionLength)) { + Record currentRecord(currentFullName); + currentRecord.destroy(); + didChange = true; + continue; + } + currentRecordStart = *(RecordIterator(currentRecordStart).operator++()); + } + if (didChange) { + notifyChangeToDelegate(); + } +} + +InternalStorage::InternalStorage() : + m_magicHeader(Magic), + m_buffer(), + m_magicFooter(Magic), + m_delegate(nullptr), + m_lastRecordRetrieved(nullptr), + m_lastRecordRetrievedPointer(nullptr) +{ + assert(m_magicHeader == Magic); + assert(m_magicFooter == Magic); + // Set the size of the first record to 0 + overrideSizeAtPosition(m_buffer, 0); +} + +// PRIVATE + +const char * InternalStorage::fullNameOfRecord(const Record record) { + char * p = pointerOfRecord(record); + if (p != nullptr) { + return fullNameOfRecordStarting(p); + } + return nullptr; +} + +InternalStorage::Record::ErrorStatus InternalStorage::setFullNameOfRecord(const Record record, const char * fullName) { + if (!FullNameCompliant(fullName)) { + return Record::ErrorStatus::NonCompliantName; + } + if (isFullNameTaken(fullName, &record)) { + return Record::ErrorStatus::NameTaken; + } + size_t nameSize = strlen(fullName) + 1; + char * p = pointerOfRecord(record); + if (p != nullptr) { + size_t previousNameSize = strlen(fullNameOfRecordStarting(p))+1; + record_size_t previousRecordSize = sizeOfRecordStarting(p); + size_t newRecordSize = previousRecordSize-previousNameSize+nameSize; + if (newRecordSize >= k_maxRecordSize || !slideBuffer(p+sizeof(record_size_t)+previousNameSize, nameSize-previousNameSize)) { + return notifyFullnessToDelegate(); + } + overrideSizeAtPosition(p, newRecordSize); + overrideFullNameAtPosition(p+sizeof(record_size_t), fullName); + notifyChangeToDelegate(record); + m_lastRecordRetrieved = record; + m_lastRecordRetrievedPointer = p; + return Record::ErrorStatus::None; + } + return Record::ErrorStatus::RecordDoesNotExist; +} + +InternalStorage::Record::ErrorStatus InternalStorage::setBaseNameWithExtensionOfRecord(Record record, const char * baseName, const char * extension) { + if (isBaseNameWithExtensionTaken(baseName, extension, &record)) { + return Record::ErrorStatus::NameTaken; + } + size_t nameSize = sizeOfBaseNameAndExtension(baseName, extension); + char * p = pointerOfRecord(record); + if (p != nullptr) { + size_t previousNameSize = strlen(fullNameOfRecordStarting(p))+1; + record_size_t previousRecordSize = sizeOfRecordStarting(p); + size_t newRecordSize = previousRecordSize-previousNameSize+nameSize; + if (newRecordSize >= k_maxRecordSize || !slideBuffer(p+sizeof(record_size_t)+previousNameSize, nameSize-previousNameSize)) { + return notifyFullnessToDelegate(); + } + overrideSizeAtPosition(p, newRecordSize); + char * fullNamePosition = p + sizeof(record_size_t); + overrideBaseNameWithExtensionAtPosition(fullNamePosition, baseName, extension); + // Recompute the CRC32 + record = Record(fullNamePosition); + notifyChangeToDelegate(record); + m_lastRecordRetrieved = record; + m_lastRecordRetrievedPointer = p; + return Record::ErrorStatus::None; + } + return Record::ErrorStatus::RecordDoesNotExist; +} + +InternalStorage::Record::Data InternalStorage::valueOfRecord(const Record record) { + char * p = pointerOfRecord(record); + if (p != nullptr) { + const char * fullName = fullNameOfRecordStarting(p); + record_size_t size = sizeOfRecordStarting(p); + const void * value = valueOfRecordStarting(p); + return {.buffer = value, .size = size-strlen(fullName)-1-sizeof(record_size_t)}; + } + return {.buffer= nullptr, .size= 0}; +} + +InternalStorage::Record::ErrorStatus InternalStorage::setValueOfRecord(Record record, Record::Data data) { + char * p = pointerOfRecord(record); + /* TODO: if data.buffer == p, assert that size hasn't change and do not do any + * memcopy, but still notify the delegate. Beware of scripts and the accordion + * routine.*/ + if (p != nullptr) { + record_size_t previousRecordSize = sizeOfRecordStarting(p); + const char * fullName = fullNameOfRecordStarting(p); + size_t newRecordSize = sizeOfRecordWithFullName(fullName, data.size); + if (newRecordSize >= k_maxRecordSize || !slideBuffer(p+previousRecordSize, newRecordSize-previousRecordSize)) { + return notifyFullnessToDelegate(); + } + record_size_t fullNameSize = strlen(fullName)+1; + overrideSizeAtPosition(p, newRecordSize); + overrideValueAtPosition(p+sizeof(record_size_t)+fullNameSize, data.buffer, data.size); + notifyChangeToDelegate(record); + m_lastRecordRetrieved = record; + m_lastRecordRetrievedPointer = p; + return Record::ErrorStatus::None; + } + return Record::ErrorStatus::RecordDoesNotExist; +} + +void InternalStorage::destroyRecord(Record record) { + if (record.isNull()) { + return; + } + char * p = pointerOfRecord(record); + if (p != nullptr) { + record_size_t previousRecordSize = sizeOfRecordStarting(p); + slideBuffer(p+previousRecordSize, -previousRecordSize); + notifyChangeToDelegate(); + } +} + +char * InternalStorage::pointerOfRecord(const Record record) const { + if (record.isNull()) { + return nullptr; + } + if (!m_lastRecordRetrieved.isNull() && record == m_lastRecordRetrieved) { + assert(m_lastRecordRetrievedPointer != nullptr); + return m_lastRecordRetrievedPointer; + } + for (char * p : *this) { + Record currentRecord(fullNameOfRecordStarting(p)); + if (record == currentRecord) { + m_lastRecordRetrieved = record; + m_lastRecordRetrievedPointer = p; + return p; + } + } + return nullptr; +} + +InternalStorage::record_size_t InternalStorage::sizeOfRecordStarting(char * start) const { + return StorageHelper::unalignedShort(start); +} + +const char * InternalStorage::fullNameOfRecordStarting(char * start) const { + return start+sizeof(record_size_t); +} + +const void * InternalStorage::valueOfRecordStarting(char * start) const { + char * currentChar = start+sizeof(record_size_t); + size_t fullNameLength = strlen(currentChar); + return currentChar+fullNameLength+1; +} + +size_t InternalStorage::overrideSizeAtPosition(char * position, record_size_t size) { + StorageHelper::writeUnalignedShort(size, position); + return sizeof(record_size_t); +} + +size_t InternalStorage::overrideFullNameAtPosition(char * position, const char * fullName) { + return strlcpy(position, fullName, strlen(fullName)+1) + 1; +} + +size_t InternalStorage::overrideBaseNameWithExtensionAtPosition(char * position, const char * baseName, const char * extension) { + size_t result = strlcpy(position, baseName, strlen(baseName)+1); // strlcpy copies the null terminating char + assert(UTF8Decoder::CharSizeOfCodePoint(k_dotChar) == 1); + *(position+result) = k_dotChar; // Replace the null terminating char with a dot + result++; + result += strlcpy(position+result, extension, strlen(extension)+1); + return result+1; +} + +size_t InternalStorage::overrideValueAtPosition(char * position, const void * data, record_size_t size) { + memcpy(position, data, size); + return size; +} + +bool InternalStorage::isFullNameTaken(const char * fullName, const Record * recordToExclude) { + Record r = Record(fullName); + return isNameOfRecordTaken(r, recordToExclude); +} + +bool InternalStorage::isBaseNameWithExtensionTaken(const char * baseName, const char * extension, Record * recordToExclude) { + Record r = Record(baseName, extension); + return isNameOfRecordTaken(r, recordToExclude); +} + +bool InternalStorage::isNameOfRecordTaken(Record r, const Record * recordToExclude) { + if (r == Record()) { + /* If the CRC32 of fullName is 0, we want to refuse the name as it would + * interfere with our escape case in the Record constructor, when the given + * name is nullptr. */ + return true; + } + for (char * p : *this) { + Record s(fullNameOfRecordStarting(p)); + if (recordToExclude && s == *recordToExclude) { + continue; + } + if (s == r) { + return true; + } + } + return false; +} + +bool InternalStorage::FullNameCompliant(const char * fullName) { + // We check that there is one dot and one dot only. + const char * dotChar = UTF8Helper::CodePointSearch(fullName, k_dotChar); + if (*dotChar == 0) { + return false; + } + if (*(UTF8Helper::CodePointSearch(dotChar+1, k_dotChar)) == 0) { + return true; + } + return false; +} + +bool InternalStorage::FullNameHasExtension(const char * fullName, const char * extension, size_t extensionLength) { + if (fullName == nullptr) { + return false; + } + size_t fullNameLength = strlen(fullName); + if (fullNameLength > extensionLength) { + const char * ext = fullName + fullNameLength - extensionLength; + if (UTF8Helper::PreviousCodePointIs(fullName, ext, k_dotChar) && strcmp(ext, extension) == 0) { + return true; + } + } + return false; +} + +char * InternalStorage::endBuffer() { + char * currentBuffer = m_buffer; + for (char * p : *this) { + currentBuffer += sizeOfRecordStarting(p); + } + return currentBuffer; +} + +size_t InternalStorage::sizeOfBaseNameAndExtension(const char * baseName, const char * extension) const { + // +1 for the dot and +1 for the null terminating char + return strlen(baseName)+1+strlen(extension)+1; +} + +size_t InternalStorage::sizeOfRecordWithBaseNameAndExtension(const char * baseName, const char * extension, size_t dataSize) const { + return sizeOfBaseNameAndExtension(baseName, extension) + dataSize + sizeof(record_size_t); +} + +size_t InternalStorage::sizeOfRecordWithFullName(const char * fullName, size_t dataSize) const { + size_t nameSize = strlen(fullName)+1; + return nameSize+dataSize+sizeof(record_size_t); +} + +bool InternalStorage::slideBuffer(char * position, int delta) { + if (delta > (int)availableSize()) { + return false; + } + memmove(position+delta, position, endBuffer()+sizeof(record_size_t)-position); + return true; +} + +InternalStorage::Record InternalStorage::privateRecordAndExtensionOfRecordBaseNamedWithExtensions(const char * baseName, const char * const extensions[], size_t numberOfExtensions, const char * * extensionResult, int baseNameLength) { + size_t nameLength = baseNameLength < 0 ? strlen(baseName) : baseNameLength; + { + const char * lastRetrievedRecordFullName = fullNameOfRecordStarting(m_lastRecordRetrievedPointer); + if (m_lastRecordRetrievedPointer != nullptr && strncmp(baseName, lastRetrievedRecordFullName, nameLength) == 0) { + for (size_t i = 0; i < numberOfExtensions; i++) { + if (strcmp(lastRetrievedRecordFullName+nameLength+1 /*+1 to pass the dot*/, extensions[i]) == 0) { + assert(UTF8Helper::CodePointIs(lastRetrievedRecordFullName + nameLength, '.')); + if (extensionResult != nullptr) { + *extensionResult = extensions[i]; + } + return m_lastRecordRetrieved; + } + } + } + } + for (char * p : *this) { + const char * currentName = fullNameOfRecordStarting(p); + if (strncmp(baseName, currentName, nameLength) == 0) { + for (size_t i = 0; i < numberOfExtensions; i++) { + if (strcmp(currentName+nameLength+1 /*+1 to pass the dot*/, extensions[i]) == 0) { + assert(UTF8Helper::CodePointIs(currentName + nameLength, '.')); + if (extensionResult != nullptr) { + *extensionResult = extensions[i]; + } + return Record(currentName); + } + } + } + } + if (extensionResult != nullptr) { + *extensionResult = nullptr; + } + return Record(); +} + +InternalStorage::RecordIterator & InternalStorage::RecordIterator::operator++() { + assert(m_recordStart); + record_size_t size = StorageHelper::unalignedShort(m_recordStart); + char * nextRecord = m_recordStart+size; + record_size_t newRecordSize = StorageHelper::unalignedShort(nextRecord); + m_recordStart = (newRecordSize == 0 ? nullptr : nextRecord); + return *this; +} + +} diff --git a/ion/src/shared/storage.cpp b/ion/src/shared/storage.cpp index 200b41865..c3dd9fcc0 100644 --- a/ion/src/shared/storage.cpp +++ b/ion/src/shared/storage.cpp @@ -1,291 +1,61 @@ -#include -#include +#include #include #include -#if ION_STORAGE_LOG -#include -#endif +#include +#include namespace Ion { -/* We want to implement a simple singleton pattern, to make sure the storage is - * initialized on first use, therefore preventing the static init order fiasco. - * That being said, we rely on knowing where the storage resides in the device's - * memory at compile time. Indeed, we want to advertise the static storage's - * memory address in the PlatformInfo structure (so that we can read and write - * it in DFU). - * Using a "static Storage storage;" variable makes it a local symbol at best, - * preventing the PlatformInfo from retrieving its address. And making the - * Storage variable global yields the static init fiasco issue. We're working - * around both issues by creating a global staticStorageArea buffer, and by - * placement-newing the Storage into that area on first use. */ - uint32_t staticStorageArea[sizeof(Storage)/sizeof(uint32_t)] = {0}; -constexpr char Storage::expExtension[]; -constexpr char Storage::funcExtension[]; -constexpr char Storage::seqExtension[]; -constexpr char Storage::eqExtension[]; - Storage * Storage::sharedStorage() { static Storage * storage = new (staticStorageArea) Storage(); return storage; } -// RECORD - -Storage::Record::Record(const char * fullName) { - if (fullName == nullptr) { - m_fullNameCRC32 = 0; - return; - } - const char * dotChar = UTF8Helper::CodePointSearch(fullName, k_dotChar); - // If no extension, return empty record - if (*dotChar == 0 || *(dotChar+1) == 0) { - m_fullNameCRC32 = 0; - return; - } - new (this) Record(fullName, dotChar - fullName, dotChar+1, (fullName + strlen(fullName)) - (dotChar+1)); -} - -Storage::Record::Record(const char * baseName, const char * extension) { - if (baseName == nullptr) { - assert(extension == nullptr); - m_fullNameCRC32 = 0; - return; - } - new (this) Record(baseName, strlen(baseName), extension, strlen(extension)); -} - - -#if ION_STORAGE_LOG - -void Storage::Record::log() { - std::cout << "Name: " << fullName() << std::endl; - std::cout << " Value (" << value().size << "): " << (char *)value().buffer << "\n\n" << std::endl; -} -#endif - -uint32_t Storage::Record::checksum() { - uint32_t crc32Results[2]; - crc32Results[0] = m_fullNameCRC32; - Data data = value(); - crc32Results[1] = Ion::crc32Byte((const uint8_t *)data.buffer, data.size); - return Ion::crc32Word(crc32Results, 2); -} - -Storage::Record::Record(const char * basename, int basenameLength, const char * extension, int extensionLength) { - assert(basename != nullptr); - assert(extension != nullptr); - - // We compute the CRC32 of the CRC32s of the basename and the extension - uint32_t crc32Results[2]; - crc32Results[0] = Ion::crc32Byte((const uint8_t *)basename, basenameLength); - crc32Results[1] = Ion::crc32Byte((const uint8_t *)extension, extensionLength); - m_fullNameCRC32 = Ion::crc32Word(crc32Results, 2); -} - -// STORAGE - -#if ION_STORAGE_LOG -void Storage::log() { - for (char * p : *this) { - const char * currentName = fullNameOfRecordStarting(p); - Record(currentName).log(); - } -} -#endif - size_t Storage::availableSize() { if (m_trashRecord != NULL) { - return realAvailableSize() + sizeof(record_size_t) + m_trashRecord.value().size; + return InternalStorage::availableSize() + sizeof(record_size_t) + m_trashRecord.value().size; } else { - return realAvailableSize(); + return InternalStorage::availableSize(); } } -size_t Storage::realAvailableSize() { - /* TODO maybe do: availableSize(char ** endBuffer) to get the endBuffer if it - * is needed after calling availableSize */ - assert(k_storageSize >= (endBuffer() - m_buffer) + sizeof(record_size_t)); - return k_storageSize-(endBuffer()-m_buffer)-sizeof(record_size_t); -} - -size_t Storage::putAvailableSpaceAtEndOfRecord(Storage::Record r) { - char * p = pointerOfRecord(r); - size_t previousRecordSize = sizeOfRecordStarting(p); - size_t availableStorageSize = realAvailableSize(); - char * nextRecord = p + previousRecordSize; - memmove(nextRecord + availableStorageSize, - nextRecord, - (m_buffer + k_storageSize - availableStorageSize) - nextRecord); - size_t newRecordSize = previousRecordSize + availableStorageSize; - overrideSizeAtPosition(p, (record_size_t)newRecordSize); - return newRecordSize; +size_t Storage::putAvailableSpaceAtEndOfRecord(Record r) { + emptyTrash(); + return InternalStorage::putAvailableSpaceAtEndOfRecord(r); } void Storage::getAvailableSpaceFromEndOfRecord(Record r, size_t recordAvailableSpace) { - char * p = pointerOfRecord(r); - size_t previousRecordSize = sizeOfRecordStarting(p); - char * nextRecord = p + previousRecordSize; - memmove(nextRecord - recordAvailableSpace, - nextRecord, - m_buffer + k_storageSize - nextRecord); - overrideSizeAtPosition(p, (record_size_t)(previousRecordSize - recordAvailableSpace)); + emptyTrash(); + InternalStorage::getAvailableSpaceFromEndOfRecord(r, recordAvailableSpace); } -uint32_t Storage::checksum() { - return Ion::crc32Byte((const uint8_t *) m_buffer, endBuffer()-m_buffer); -} - -void Storage::notifyChangeToDelegate(const Record record) const { - m_lastRecordRetrieved = Record(nullptr); - m_lastRecordRetrievedPointer = nullptr; - if (m_delegate != nullptr) { - m_delegate->storageDidChangeForRecord(record); +int Storage::numberOfRecordsWithExtension(const char * extension) { + int trashRecord = 0; + if (FullNameHasExtension(m_trashRecord.fullName(), extension, strlen(extension))) { + trashRecord = 1; } -} - -Storage::Record::ErrorStatus Storage::notifyFullnessToDelegate() const { - if (m_delegate != nullptr) { - m_delegate->storageIsFull(); - } - return Record::ErrorStatus::NotEnoughSpaceAvailable; + return InternalStorage::numberOfRecordsWithExtension(extension) - trashRecord; } Storage::Record::ErrorStatus Storage::createRecordWithFullName(const char * fullName, const void * data, size_t size) { emptyTrash(); - size_t recordSize = sizeOfRecordWithFullName(fullName, size); - if (recordSize >= k_maxRecordSize || recordSize > availableSize()) { - return notifyFullnessToDelegate(); - } - if (isFullNameTaken(fullName)) { - return Record::ErrorStatus::NameTaken; - } - // Find the end of data - char * newRecordAddress = endBuffer(); - char * newRecord = newRecordAddress; - // Fill totalSize - newRecord += overrideSizeAtPosition(newRecord, (record_size_t)recordSize); - // Fill name - newRecord += overrideFullNameAtPosition(newRecord, fullName); - // Fill data - newRecord += overrideValueAtPosition(newRecord, data, size); - // Next Record is null-sized - overrideSizeAtPosition(newRecord, 0); - Record r = Record(fullName); - notifyChangeToDelegate(r); - m_lastRecordRetrieved = r; - m_lastRecordRetrievedPointer = newRecordAddress; - return Record::ErrorStatus::None; + return InternalStorage::createRecordWithFullName(fullName, data, size); } Storage::Record::ErrorStatus Storage::createRecordWithExtension(const char * baseName, const char * extension, const void * data, size_t size) { emptyTrash(); - size_t recordSize = sizeOfRecordWithBaseNameAndExtension(baseName, extension, size); - if (recordSize >= k_maxRecordSize || recordSize > availableSize()) { - return notifyFullnessToDelegate(); - } - if (isBaseNameWithExtensionTaken(baseName, extension)) { - return Record::ErrorStatus::NameTaken; - } - // Find the end of data - char * newRecordAddress = endBuffer(); - char * newRecord = newRecordAddress; - // Fill totalSize - newRecord += overrideSizeAtPosition(newRecord, (record_size_t)recordSize); - // Fill name - newRecord += overrideBaseNameWithExtensionAtPosition(newRecord, baseName, extension); - // Fill data - newRecord += overrideValueAtPosition(newRecord, data, size); - // Next Record is null-sized - overrideSizeAtPosition(newRecord, 0); - Record r = Record(fullNameOfRecordStarting(newRecordAddress)); - notifyChangeToDelegate(r); - m_lastRecordRetrieved = r; - m_lastRecordRetrievedPointer = newRecordAddress; - return Record::ErrorStatus::None; + return InternalStorage::createRecordWithExtension(baseName, extension, data, size); } -int Storage::numberOfRecordsWithExtension(const char * extension) { - int count = 0; - size_t extensionLength = strlen(extension); - for (char * p : *this) { - const char * name = fullNameOfRecordStarting(p); - if (FullNameHasExtension(name, extension, extensionLength) && Record(name) != m_trashRecord) { - count++; - } - } - return count; +bool Storage::hasRecord(Record r) { + return InternalStorage::hasRecord(r) && r != m_trashRecord; } -int Storage::numberOfRecords() { - return realNumberOfRecords() - (m_trashRecord == NULL ? 0 : 1); -} - -int Storage::realNumberOfRecords() { - int count = 0; - for (char * p : *this) { - const char * name = fullNameOfRecordStarting(p); - count++; - } - return count; -} - -Storage::Record Storage::realRecordAtIndex(int index) { - int currentIndex = -1; - const char * name = nullptr; - char * recordAddress = nullptr; - for (char * p : *this) { - const char * currentName = fullNameOfRecordStarting(p); - currentIndex++; - if (currentIndex == index) { - recordAddress = p; - name = currentName; - break; - } - } - if (name == nullptr) { - return Record(); - } - Record r = Record(name); - m_lastRecordRetrieved = r; - m_lastRecordRetrievedPointer = recordAddress; - return Record(name); -} - - -Storage::Record Storage::recordAtIndex(int index) { - int currentIndex = -1; - const char * name = nullptr; - char * recordAddress = nullptr; - for (char * p : *this) { - const char * currentName = fullNameOfRecordStarting(p); - Record r = Record(currentName); - if (r == m_trashRecord) { - continue; - } - currentIndex++; - if (currentIndex == index) { - recordAddress = p; - name = currentName; - break; - } - } - if (name == nullptr) { - return Record(); - } - Record r = Record(name); - m_lastRecordRetrieved = r; - m_lastRecordRetrievedPointer = recordAddress; - return Record(name); -} - -void Storage::emptyTrash() { - if (m_trashRecord != NULL) { - realDestroyRecord(m_trashRecord); - m_trashRecord = NULL; - } +void Storage::destroyRecord(Record record) { + emptyTrash(); + m_trashRecord = record; } Storage::Record Storage::recordWithExtensionAtIndex(const char * extension, int index) { @@ -307,397 +77,98 @@ Storage::Record Storage::recordWithExtensionAtIndex(const char * extension, int if (name == nullptr) { return Record(); } - Record r = Record(name); + Storage::Record r = Record(name); m_lastRecordRetrieved = r; m_lastRecordRetrievedPointer = recordAddress; return Record(name); } Storage::Record Storage::recordNamed(const char * fullName) { - if (fullName == nullptr) { - return Record(); - } - Record r = Record(fullName); - char * p = pointerOfRecord(r); - if (p != nullptr) { - return r; - } - return Record(); + Storage::Record r = InternalStorage::recordNamed(fullName); + return r == m_trashRecord ? Record() : r; } Storage::Record Storage::recordBaseNamedWithExtension(const char * baseName, const char * extension) { - const char * extensions[1] = {extension}; - return recordBaseNamedWithExtensions(baseName, extensions, 1); + Storage::Record r = InternalStorage::recordBaseNamedWithExtension(baseName, extension); + return r == m_trashRecord ? Record() : r; } -Storage::Record Storage::recordBaseNamedWithExtensions(const char * baseName, const char * const extensions[], size_t numberOfExtensions) { - return privateRecordAndExtensionOfRecordBaseNamedWithExtensions(baseName, extensions, numberOfExtensions); +Storage::Record Storage::recordBaseNamedWithExtensions(const char * baseName, const char * const extension[], size_t numberOfExtensions) { + Storage::Record r = InternalStorage::recordBaseNamedWithExtensions(baseName, extension, numberOfExtensions); + return r == m_trashRecord ? Record() : r; } -const char * Storage::extensionOfRecordBaseNamedWithExtensions(const char * baseName, int baseNameLength, const char * const extensions[], size_t numberOfExtensions) { - const char * result = nullptr; - privateRecordAndExtensionOfRecordBaseNamedWithExtensions(baseName, extensions, numberOfExtensions, &result, baseNameLength); - return result; -} - -void Storage::destroyAllRecords() { - overrideSizeAtPosition(m_buffer, 0); - notifyChangeToDelegate(); -} - -void Storage::destroyRecordWithBaseNameAndExtension(const char * baseName, const char * extension) { - recordBaseNamedWithExtension(baseName, extension).destroy(); -} - -void Storage::destroyRecordsWithExtension(const char * extension) { - size_t extensionLength = strlen(extension); - char * currentRecordStart = (char *)m_buffer; - bool didChange = false; - while (currentRecordStart != nullptr && sizeOfRecordStarting(currentRecordStart) != 0) { - const char * currentFullName = fullNameOfRecordStarting(currentRecordStart); - if (FullNameHasExtension(currentFullName, extension, extensionLength)) { - Record currentRecord(currentFullName); - currentRecord.destroy(); - didChange = true; - continue; - } - currentRecordStart = *(RecordIterator(currentRecordStart).operator++()); - } - if (didChange) { - notifyChangeToDelegate(); - } -} - -// PRIVATE - -Storage::Storage() : - m_magicHeader(Magic), - m_buffer(), - m_magicFooter(Magic), - m_delegate(nullptr), - m_lastRecordRetrieved(nullptr), - m_lastRecordRetrievedPointer(nullptr), - m_trashRecord(NULL) -{ - assert(m_magicHeader == Magic); - assert(m_magicFooter == Magic); - // Set the size of the first record to 0 - overrideSizeAtPosition(m_buffer, 0); -} - -const char * Storage::fullNameOfRecord(const Record record) { - char * p = pointerOfRecord(record); - if (p != nullptr) { - return fullNameOfRecordStarting(p); - } - return nullptr; -} - -Storage::Record::ErrorStatus Storage::setFullNameOfRecord(const Record record, const char * fullName) { - if (!FullNameCompliant(fullName)) { - return Record::ErrorStatus::NonCompliantName; - } - if (isFullNameTaken(fullName, &record)) { - return Record::ErrorStatus::NameTaken; - } - size_t nameSize = strlen(fullName) + 1; - char * p = pointerOfRecord(record); - if (p != nullptr) { - size_t previousNameSize = strlen(fullNameOfRecordStarting(p))+1; - record_size_t previousRecordSize = sizeOfRecordStarting(p); - size_t newRecordSize = previousRecordSize-previousNameSize+nameSize; - if (newRecordSize >= k_maxRecordSize || !slideBuffer(p+sizeof(record_size_t)+previousNameSize, nameSize-previousNameSize)) { - return notifyFullnessToDelegate(); - } - overrideSizeAtPosition(p, newRecordSize); - overrideFullNameAtPosition(p+sizeof(record_size_t), fullName); - notifyChangeToDelegate(record); - m_lastRecordRetrieved = record; - m_lastRecordRetrievedPointer = p; - return Record::ErrorStatus::None; - } - return Record::ErrorStatus::RecordDoesNotExist; -} - -Storage::Record::ErrorStatus Storage::setBaseNameWithExtensionOfRecord(Record record, const char * baseName, const char * extension) { - if (isBaseNameWithExtensionTaken(baseName, extension, &record)) { - return Record::ErrorStatus::NameTaken; - } - size_t nameSize = sizeOfBaseNameAndExtension(baseName, extension); - char * p = pointerOfRecord(record); - if (p != nullptr) { - size_t previousNameSize = strlen(fullNameOfRecordStarting(p))+1; - record_size_t previousRecordSize = sizeOfRecordStarting(p); - size_t newRecordSize = previousRecordSize-previousNameSize+nameSize; - if (newRecordSize >= k_maxRecordSize || !slideBuffer(p+sizeof(record_size_t)+previousNameSize, nameSize-previousNameSize)) { - return notifyFullnessToDelegate(); - } - overrideSizeAtPosition(p, newRecordSize); - char * fullNamePosition = p + sizeof(record_size_t); - overrideBaseNameWithExtensionAtPosition(fullNamePosition, baseName, extension); - // Recompute the CRC32 - record = Record(fullNamePosition); - notifyChangeToDelegate(record); - m_lastRecordRetrieved = record; - m_lastRecordRetrievedPointer = p; - return Record::ErrorStatus::None; - } - return Record::ErrorStatus::RecordDoesNotExist; -} - -Storage::Record::Data Storage::valueOfRecord(const Record record) { - char * p = pointerOfRecord(record); - if (p != nullptr) { - const char * fullName = fullNameOfRecordStarting(p); - record_size_t size = sizeOfRecordStarting(p); - const void * value = valueOfRecordStarting(p); - return {.buffer = value, .size = size-strlen(fullName)-1-sizeof(record_size_t)}; - } - return {.buffer= nullptr, .size= 0}; -} - -Storage::Record::ErrorStatus Storage::setValueOfRecord(Record record, Record::Data data) { - char * p = pointerOfRecord(record); - /* TODO: if data.buffer == p, assert that size hasn't change and do not do any - * memcopy, but still notify the delegate. Beware of scripts and the accordion - * routine.*/ - if (p != nullptr) { - record_size_t previousRecordSize = sizeOfRecordStarting(p); - const char * fullName = fullNameOfRecordStarting(p); - size_t newRecordSize = sizeOfRecordWithFullName(fullName, data.size); - if (newRecordSize >= k_maxRecordSize || !slideBuffer(p+previousRecordSize, newRecordSize-previousRecordSize)) { - return notifyFullnessToDelegate(); - } - record_size_t fullNameSize = strlen(fullName)+1; - overrideSizeAtPosition(p, newRecordSize); - overrideValueAtPosition(p+sizeof(record_size_t)+fullNameSize, data.buffer, data.size); - notifyChangeToDelegate(record); - m_lastRecordRetrieved = record; - m_lastRecordRetrievedPointer = p; - return Record::ErrorStatus::None; - } - return Record::ErrorStatus::RecordDoesNotExist; -} - - -void Storage::reinsertTrash(const char * extension) { - if (m_trashRecord != NULL) { - char * p = pointerOfRecord(m_trashRecord); - const char * fullName = fullNameOfRecordStarting(p); - if (FullNameHasExtension(fullName, extension, strlen(extension))) { - m_trashRecord = NULL; - } - } -} - - -void Storage::destroyRecord(Record record) { - emptyTrash(); - m_trashRecord = record; -} - -void Storage::realDestroyRecord(Record record) { - if (record.isNull()) { - return; - } - char * p = pointerOfRecord(record); - if (p != nullptr) { - record_size_t previousRecordSize = sizeOfRecordStarting(p); - slideBuffer(p+previousRecordSize, -previousRecordSize); - notifyChangeToDelegate(); - } -} - -char * Storage::pointerOfRecord(const Record record) const { - if (record.isNull()) { - return nullptr; - } - if (!m_lastRecordRetrieved.isNull() && record == m_lastRecordRetrieved) { - assert(m_lastRecordRetrievedPointer != nullptr); - return m_lastRecordRetrievedPointer; - } - for (char * p : *this) { - Record currentRecord(fullNameOfRecordStarting(p)); - if (record == currentRecord) { - m_lastRecordRetrieved = record; - m_lastRecordRetrievedPointer = p; - return p; - } - } - return nullptr; -} - -Storage::record_size_t Storage::sizeOfRecordStarting(char * start) const { - return StorageHelper::unalignedShort(start); -} - -const char * Storage::fullNameOfRecordStarting(char * start) const { - return start+sizeof(record_size_t); -} - -const void * Storage::valueOfRecordStarting(char * start) const { - char * currentChar = start+sizeof(record_size_t); - size_t fullNameLength = strlen(currentChar); - return currentChar+fullNameLength+1; -} - -size_t Storage::overrideSizeAtPosition(char * position, record_size_t size) { - StorageHelper::writeUnalignedShort(size, position); - return sizeof(record_size_t); -} - -size_t Storage::overrideFullNameAtPosition(char * position, const char * fullName) { - return strlcpy(position, fullName, strlen(fullName)+1) + 1; -} - -size_t Storage::overrideBaseNameWithExtensionAtPosition(char * position, const char * baseName, const char * extension) { - size_t result = strlcpy(position, baseName, strlen(baseName)+1); // strlcpy copies the null terminating char - assert(UTF8Decoder::CharSizeOfCodePoint(k_dotChar) == 1); - *(position+result) = k_dotChar; // Replace the null terminating char with a dot - result++; - result += strlcpy(position+result, extension, strlen(extension)+1); - return result+1; -} - -size_t Storage::overrideValueAtPosition(char * position, const void * data, record_size_t size) { - memcpy(position, data, size); - return size; -} - -bool Storage::isFullNameTaken(const char * fullName, const Record * recordToExclude) { - Record r = Record(fullName); - return isNameOfRecordTaken(r, recordToExclude); -} - -bool Storage::isBaseNameWithExtensionTaken(const char * baseName, const char * extension, Record * recordToExclude) { - Record r = Record(baseName, extension); - return isNameOfRecordTaken(r, recordToExclude); -} - -bool Storage::isNameOfRecordTaken(Record r, const Record * recordToExclude) { - if (r == Record()) { - /* If the CRC32 of fullName is 0, we want to refuse the name as it would - * interfere with our escape case in the Record constructor, when the given - * name is nullptr. */ - return true; - } - for (char * p : *this) { - Record s(fullNameOfRecordStarting(p)); - if (recordToExclude && s == *recordToExclude) { - continue; - } - if (s == r && s != m_trashRecord) { - return true; - } - } - return false; -} - -bool Storage::FullNameCompliant(const char * fullName) { - // We check that there is one dot and one dot only. - const char * dotChar = UTF8Helper::CodePointSearch(fullName, k_dotChar); - if (*dotChar == 0) { - return false; - } - if (*(UTF8Helper::CodePointSearch(dotChar+1, k_dotChar)) == 0) { - return true; - } - return false; -} - -bool Storage::FullNameHasExtension(const char * fullName, const char * extension, size_t extensionLength) { - if (fullName == nullptr) { - return false; - } - size_t fullNameLength = strlen(fullName); - if (fullNameLength > extensionLength) { - const char * ext = fullName + fullNameLength - extensionLength; - if (UTF8Helper::PreviousCodePointIs(fullName, ext, k_dotChar) && strcmp(ext, extension) == 0) { - return true; - } - } - return false; -} - -char * Storage::endBuffer() { - char * currentBuffer = m_buffer; - for (char * p : *this) { - currentBuffer += sizeOfRecordStarting(p); - } - return currentBuffer; -} - -size_t Storage::sizeOfBaseNameAndExtension(const char * baseName, const char * extension) const { - // +1 for the dot and +1 for the null terminating char - return strlen(baseName)+1+strlen(extension)+1; -} - -size_t Storage::sizeOfRecordWithBaseNameAndExtension(const char * baseName, const char * extension, size_t dataSize) const { - return sizeOfBaseNameAndExtension(baseName, extension) + dataSize + sizeof(record_size_t); -} - -size_t Storage::sizeOfRecordWithFullName(const char * fullName, size_t dataSize) const { - size_t nameSize = strlen(fullName)+1; - return nameSize+dataSize+sizeof(record_size_t); -} - -bool Storage::slideBuffer(char * position, int delta) { - if (delta > (int)realAvailableSize()) { - emptyTrash(); - if (delta > (int)realAvailableSize()) { - return false; - } - } - memmove(position+delta, position, endBuffer()+sizeof(record_size_t)-position); - return true; -} - -Storage::Record Storage::privateRecordAndExtensionOfRecordBaseNamedWithExtensions(const char * baseName, const char * const extensions[], size_t numberOfExtensions, const char * * extensionResult, int baseNameLength) { +const char * Storage::extensionOfRecordBaseNamedWithExtensions(const char * baseName, int baseNameLength, const char * const extension[], size_t numberOfExtensions) { size_t nameLength = baseNameLength < 0 ? strlen(baseName) : baseNameLength; { const char * lastRetrievedRecordFullName = fullNameOfRecordStarting(m_lastRecordRetrievedPointer); - if (m_lastRecordRetrievedPointer != nullptr && strncmp(baseName, lastRetrievedRecordFullName, nameLength) == 0 && Record(lastRetrievedRecordFullName) == m_trashRecord) { + if (m_lastRecordRetrievedPointer != nullptr && strncmp(baseName, lastRetrievedRecordFullName, nameLength) == 0 && Record(lastRetrievedRecordFullName) != m_trashRecord) { for (size_t i = 0; i < numberOfExtensions; i++) { - if (strcmp(lastRetrievedRecordFullName+nameLength+1 /*+1 to pass the dot*/, extensions[i]) == 0) { + if (strcmp(lastRetrievedRecordFullName+nameLength+1 /*+1 to pass the dot*/, extension[i]) == 0) { assert(UTF8Helper::CodePointIs(lastRetrievedRecordFullName + nameLength, '.')); - if (extensionResult != nullptr) { - *extensionResult = extensions[i]; - } - return m_lastRecordRetrieved; + return extension[i]; } } } } for (char * p : *this) { const char * currentName = fullNameOfRecordStarting(p); - if (strncmp(baseName, currentName, nameLength) == 0) { - if (Record(currentName) == m_trashRecord) { - continue; - } + if (strncmp(baseName, currentName, nameLength) == 0 && Record(currentName) != m_trashRecord) { for (size_t i = 0; i < numberOfExtensions; i++) { - if (strcmp(currentName+nameLength+1 /*+1 to pass the dot*/, extensions[i]) == 0) { + if (strcmp(currentName+nameLength+1 /*+1 to pass the dot*/, extension[i]) == 0) { assert(UTF8Helper::CodePointIs(currentName + nameLength, '.')); - if (extensionResult != nullptr) { - *extensionResult = extensions[i]; - } - return Record(currentName); + return extension[i]; } } } } - if (extensionResult != nullptr) { - *extensionResult = nullptr; + + return nullptr; +} + +int Storage::numberOfRecords() { + return InternalStorage::numberOfRecords() - (m_trashRecord != NULL ? 1 : 0); +} + +InternalStorage::Record Storage::recordAtIndex(int index) { + int currentIndex = -1; + const char * name = nullptr; + char * recordAddress = nullptr; + for (char * p : *this) { + const char * currentName = fullNameOfRecordStarting(p); + if (Record(currentName) != m_trashRecord) { + currentIndex++; + if (currentIndex == index) { + recordAddress = p; + name = currentName; + break; + } + } } - return Record(); + if (name == nullptr) { + return Record(); + } + Record r = Record(name); + m_lastRecordRetrieved = r; + m_lastRecordRetrievedPointer = recordAddress; + return Record(name); } -Storage::RecordIterator & Storage::RecordIterator::operator++() { - assert(m_recordStart); - record_size_t size = StorageHelper::unalignedShort(m_recordStart); - char * nextRecord = m_recordStart+size; - record_size_t newRecordSize = StorageHelper::unalignedShort(nextRecord); - m_recordStart = (newRecordSize == 0 ? nullptr : nextRecord); - return *this; +void Storage::reinsertTrash(const char * extension) { + if (!m_trashRecord.isNull()) { + char * p = pointerOfRecord(m_trashRecord); + const char * fullName = fullNameOfRecordStarting(p); + if (FullNameHasExtension(fullName, extension, strlen(extension))) { + m_trashRecord = Record(); + } + } +} + +void Storage::emptyTrash() { + if (!m_trashRecord.isNull()) { + InternalStorage::destroyRecord(m_trashRecord); + m_trashRecord = Record(); + } } } diff --git a/python/port/mod/ion/file.cpp b/python/port/mod/ion/file.cpp index 721d79eee..c9eb9bc66 100644 --- a/python/port/mod/ion/file.cpp +++ b/python/port/mod/ion/file.cpp @@ -725,12 +725,12 @@ STATIC mp_obj_t file_write(mp_obj_t o_in, mp_obj_t o_s) { size_t previous_size = file->record.value().size; - // Claim avaliable space. - size_t avaliable_size = Ion::Storage::sharedStorage()->putAvailableSpaceAtEndOfRecord(file->record); + // Claim available space. + size_t available_size = Ion::Storage::sharedStorage()->putAvailableSpaceAtEndOfRecord(file->record); // Check if there is enough space left - if (file->position + len > avaliable_size) { - Ion::Storage::sharedStorage()->getAvailableSpaceFromEndOfRecord(file->record, avaliable_size - previous_size); + if (file->position + len > available_size) { + Ion::Storage::sharedStorage()->getAvailableSpaceFromEndOfRecord(file->record, available_size - previous_size); mp_raise_OSError(28); } @@ -974,12 +974,12 @@ STATIC mp_obj_t file_truncate(size_t n_args, const mp_obj_t* args) { size_t previous_size = file->record.value().size; - // Claim avaliable space. - size_t avaliable_size = Ion::Storage::sharedStorage()->putAvailableSpaceAtEndOfRecord(file->record); + // Claim available space. + size_t available_size = Ion::Storage::sharedStorage()->putAvailableSpaceAtEndOfRecord(file->record); // Check if there is enough space left - if (new_end > avaliable_size) { - Ion::Storage::sharedStorage()->getAvailableSpaceFromEndOfRecord(file->record, avaliable_size - previous_size); + if (new_end > available_size) { + Ion::Storage::sharedStorage()->getAvailableSpaceFromEndOfRecord(file->record, available_size - previous_size); mp_raise_OSError(28); }