From 922e0979e2c27ff8dbc765ae151d17c7815b98a0 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Tue, 26 Jun 2018 11:44:11 +0200 Subject: [Storage] Implement Key API --- common/changereplay.cpp | 4 +- common/storage.h | 156 ++++++++++++++++++++++++++++++++++++++++- common/storage/entitystore.cpp | 66 ++++++++++++----- common/storage_common.cpp | 29 +++++++- common/synchronizer.cpp | 3 +- common/typeindex.cpp | 9 --- common/utils.cpp | 1 + common/utils.h | 13 +++- 8 files changed, 245 insertions(+), 36 deletions(-) (limited to 'common') diff --git a/common/changereplay.cpp b/common/changereplay.cpp index 0adbd78..e81f52c 100644 --- a/common/changereplay.cpp +++ b/common/changereplay.cpp @@ -113,7 +113,9 @@ KAsync::Job ChangeReplay::replayNextRevision() if (uid.isEmpty() || type.isEmpty()) { SinkErrorCtx(mLogCtx) << "Failed to read uid or type for revison: " << revision << uid << type; } else { - const auto key = DataStore::assembleKey(uid, revision); + //const auto key = DataStore::assembleKey(uid, revision); + // TODO: should not use internal representations + const auto key = Storage::Key(Storage::Identifier::fromDisplayByteArray(uid), revision).toInternalByteArray(); QByteArray entityBuffer; DataStore::mainDatabase(mMainStoreTransaction, type) .scan(key, diff --git a/common/storage.h b/common/storage.h index a8c486c..3cc5adf 100644 --- a/common/storage.h +++ b/common/storage.h @@ -22,8 +22,10 @@ #pragma once #include "sink_export.h" +#include "utils.h" #include #include +#include #include #include @@ -38,6 +40,151 @@ struct SINK_EXPORT DbLayout { Databases tables; }; +class Identifier +{ +public: + // RFC 4122 Section 4.1.2 says 128 bits -> 16 bytes + static const constexpr size_t INTERNAL_REPR_SIZE = 16; + static const constexpr size_t DISPLAY_REPR_SIZE = 38; + + Identifier() : uid(QUuid::createUuid()) {}; + + QByteArray toInternalByteArray() const + { + return uid.toRfc4122(); + } + + static Identifier fromInternalByteArray(const QByteArray &bytes) + { + Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); + return Identifier(QUuid::fromRfc4122(bytes)); + } + + QString toDisplayString() const + { + return uid.toString(); + } + + QByteArray toDisplayByteArray() const + { + return uid.toByteArray(); + } + + static Identifier fromDisplayByteArray(const QByteArray &bytes) + { + Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); + return Identifier(QUuid::fromString(QString::fromUtf8(bytes))); + } + +private: + explicit Identifier(const QUuid &uid) : uid(uid) {} + QUuid uid; +}; + +class Revision +{ +public: + // qint64 has a 19 digit decimal representation + static const constexpr size_t INTERNAL_REPR_SIZE = 19; + static const constexpr size_t DISPLAY_REPR_SIZE = 19; + + Revision(qint64 rev) : rev(rev) {} + + QByteArray toInternalByteArray() const + { + return padNumber(rev); + } + + static Revision fromInternalByteArray(const QByteArray &bytes) + { + Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); + return Revision(bytes.toLongLong()); + } + + QString toDisplayString() const + { + return QString::fromUtf8(toInternalByteArray()); + } + + QByteArray toDisplayByteArray() const + { + return toInternalByteArray(); + } + + static Revision fromDisplayByteArray(const QByteArray &bytes) + { + Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); + return fromInternalByteArray(bytes); + } + + qint64 toQint64() const + { + return rev; + } + +private: + qint64 rev; +}; + +class Key +{ +public: + static const constexpr size_t INTERNAL_REPR_SIZE = Identifier::INTERNAL_REPR_SIZE + Revision::INTERNAL_REPR_SIZE; + static const constexpr size_t DISPLAY_REPR_SIZE = Identifier::DISPLAY_REPR_SIZE + Revision::DISPLAY_REPR_SIZE; + + Key(const Identifier &id, const Revision &rev) : id(id), rev(rev) {} + + QByteArray toInternalByteArray() const + { + return id.toInternalByteArray() + rev.toInternalByteArray(); + } + + static Key fromInternalByteArray(const QByteArray &bytes) + { + Q_ASSERT(bytes.size() == INTERNAL_REPR_SIZE); + auto idBytes = bytes.mid(0, Identifier::INTERNAL_REPR_SIZE); + auto revBytes = bytes.mid(Identifier::INTERNAL_REPR_SIZE); + return Key(Identifier::fromInternalByteArray(idBytes), Revision::fromInternalByteArray(revBytes)); + } + + QString toDisplayString() const + { + return id.toDisplayString() + rev.toDisplayString(); + } + + QByteArray toDisplayByteArray() const + { + return id.toDisplayByteArray() + rev.toDisplayByteArray(); + } + + static Key fromDisplayByteArray(const QByteArray &bytes) + { + Q_ASSERT(bytes.size() == DISPLAY_REPR_SIZE); + auto idBytes = bytes.mid(0, Identifier::DISPLAY_REPR_SIZE); + auto revBytes = bytes.mid(Identifier::DISPLAY_REPR_SIZE); + return Key(Identifier::fromDisplayByteArray(idBytes), Revision::fromDisplayByteArray(revBytes)); + } + + const Identifier &identifier() const + { + return id; + } + + const Revision &revision() const + { + return rev; + } + + void setRevision(const Revision &newRev) + { + rev = newRev; + } + +private: + Identifier id; + Revision rev; +}; + class SINK_EXPORT DataStore { public: @@ -237,9 +384,9 @@ public: static bool isInternalKey(void *key, int keySize); static bool isInternalKey(const QByteArray &key); - static QByteArray assembleKey(const QByteArray &key, qint64 revision); - static QByteArray uidFromKey(const QByteArray &key); - static qint64 revisionFromKey(const QByteArray &key); + //static QByteArray assembleKey(const QByteArray &key, qint64 revision); + //static Identifier uidFromKey(const QByteArray &key); + //static qint64 revisionFromKey(const QByteArray &key); static NamedDatabase mainDatabase(const Transaction &, const QByteArray &type); @@ -260,3 +407,6 @@ private: } // namespace Sink SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::DataStore::Error &error); +SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Identifier &); +SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Revision &); +SINK_EXPORT QDebug& operator<<(QDebug &dbg, const Sink::Storage::Key &); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index dd6bbf0..f74d3df 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -237,8 +237,10 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool flatbuffers::FlatBufferBuilder fbb; d->resourceContext.adaptorFactory(type).createBuffer(entity, fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize()); + const auto key = Key(Identifier::fromDisplayByteArray(entity.identifier()), newRevision); + DataStore::mainDatabase(d->transaction, type) - .write(DataStore::assembleKey(entity.identifier(), newRevision), BufferUtils::extractBuffer(fbb), + .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << entity.identifier() << newRevision; }); DataStore::setMaxRevision(d->transaction, newRevision); DataStore::recordRevision(d->transaction, newRevision, entity.identifier(), type); @@ -311,8 +313,10 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu flatbuffers::FlatBufferBuilder fbb; d->resourceContext.adaptorFactory(type).createBuffer(newEntity, fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize()); + const auto key = Key(Identifier::fromDisplayByteArray(newEntity.identifier()), newRevision); + DataStore::mainDatabase(d->transaction, type) - .write(DataStore::assembleKey(newEntity.identifier(), newRevision), BufferUtils::extractBuffer(fbb), + .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << newEntity.identifier() << newRevision; }); DataStore::setMaxRevision(d->transaction, newRevision); DataStore::recordRevision(d->transaction, newRevision, newEntity.identifier(), type); @@ -346,8 +350,10 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu flatbuffers::FlatBufferBuilder fbb; EntityBuffer::assembleEntityBuffer(fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize(), 0, 0, 0, 0); + const auto key = Key(Identifier::fromDisplayByteArray(uid), newRevision); + DataStore::mainDatabase(d->transaction, type) - .write(DataStore::assembleKey(uid, newRevision), BufferUtils::extractBuffer(fbb), + .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << uid << newRevision; }); DataStore::setMaxRevision(d->transaction, newRevision); DataStore::recordRevision(d->transaction, newRevision, uid, type); @@ -428,7 +434,7 @@ QVector EntityStore::fullScan(const QByteArray &type) DataStore::mainDatabase(d->getTransaction(), type) .scan(QByteArray(), [&](const QByteArray &key, const QByteArray &value) -> bool { - const auto uid = DataStore::uidFromKey(key); + const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); if (keys.contains(uid)) { //Not something that should persist if the replay works, so we keep a message for now. SinkTraceCtx(d->logCtx) << "Multiple revisions for key: " << key; @@ -479,16 +485,24 @@ void EntityStore::indexLookup(const QByteArray &type, const QByteArray &property /* }); */ } -void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) +void EntityStore::readLatest(const QByteArray &type, const QByteArray &key, const std::function callback) { Q_ASSERT(d); - Q_ASSERT(!uid.isEmpty()); + Q_ASSERT(!key.isEmpty()); + const auto internalKey = [&key]() { + if(key.size() == Identifier::DISPLAY_REPR_SIZE) { + return Identifier::fromDisplayByteArray(key).toInternalByteArray(); + } else { + return Key::fromDisplayByteArray(key).toInternalByteArray(); + } + }(); auto db = DataStore::mainDatabase(d->getTransaction(), type); - db.findLatest(uid, + db.findLatest(internalKey, [=](const QByteArray &key, const QByteArray &value) { - callback(DataStore::uidFromKey(key), Sink::EntityBuffer(value.data(), value.size())); + const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); + callback(uid, Sink::EntityBuffer(value.data(), value.size())); }, - [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << uid; }); + [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << key; }); } void EntityStore::readLatest(const QByteArray &type, const QByteArray &uid, const std::function callback) @@ -516,12 +530,15 @@ ApplicationDomain::ApplicationDomainType EntityStore::readLatest(const QByteArra return dt; } -void EntityStore::readEntity(const QByteArray &type, const QByteArray &key, const std::function callback) +// TODO: check every usage +void EntityStore::readEntity(const QByteArray &type, const QByteArray &displayKey, const std::function callback) { + const auto key = Key::fromDisplayByteArray(displayKey); auto db = DataStore::mainDatabase(d->getTransaction(), type); - db.scan(key, + db.scan(key.toInternalByteArray(), [=](const QByteArray &key, const QByteArray &value) -> bool { - callback(DataStore::uidFromKey(key), Sink::EntityBuffer(value.data(), value.size())); + const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); + callback(uid, Sink::EntityBuffer(value.data(), value.size())); return false; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readEntity query: " << error.message << key; }); @@ -567,9 +584,10 @@ void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedT revisionCounter++; continue; } - const auto key = DataStore::assembleKey(uid, revisionCounter); + const auto key = Key(Identifier::fromDisplayByteArray(uid), revisionCounter); + revisionCounter++; - callback(key); + callback(key.toDisplayByteArray()); } } @@ -577,16 +595,19 @@ void EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qi { auto db = DataStore::mainDatabase(d->getTransaction(), type); qint64 latestRevision = 0; - db.scan(uid, + const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); + db.scan(internalUid, [&latestRevision, revision](const QByteArray &key, const QByteArray &) -> bool { - const auto foundRevision = DataStore::revisionFromKey(key); + //const auto foundRevision = DataStore::revisionFromKey(key); + const auto foundRevision = Key::fromInternalByteArray(key).revision().toQint64(); if (foundRevision < revision && foundRevision > latestRevision) { latestRevision = foundRevision; } return true; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read current value from storage: " << error.message; }, true); - readEntity(type, DataStore::assembleKey(uid, latestRevision), callback); + const auto key = Key(Identifier::fromDisplayByteArray(uid), latestRevision); + readEntity(type, key.toDisplayByteArray(), callback); } void EntityStore::readPrevious(const QByteArray &type, const QByteArray &uid, qint64 revision, const std::function callback) @@ -612,15 +633,22 @@ void EntityStore::readAllUids(const QByteArray &type, const std::functiongetTransaction(), type).contains(uid); + // Because of pipeline using this function with new entities + // TODO: maybe modify pipeline instead? + if(uid.isEmpty()) { + return false; + } + const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); + return DataStore::mainDatabase(d->getTransaction(), type).contains(internalUid); } bool EntityStore::exists(const QByteArray &type, const QByteArray &uid) { bool found = false; bool alreadyRemoved = false; + const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); DataStore::mainDatabase(d->transaction, type) - .findLatest(uid, + .findLatest(internalUid, [&found, &alreadyRemoved](const QByteArray &key, const QByteArray &data) { auto entity = GetEntity(data.data()); if (entity && entity->metadata()) { diff --git a/common/storage_common.cpp b/common/storage_common.cpp index 057dce4..b36ffcb 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -24,18 +24,40 @@ #include "log.h" #include "utils.h" +#include + QDebug& operator<<(QDebug &dbg, const Sink::Storage::DataStore::Error &error) { dbg << error.message << "Code: " << error.code << "Db: " << error.store; return dbg; } +QDebug& operator<<(QDebug &dbg, const Sink::Storage::Identifier &id) +{ + dbg << id.toDisplayString(); + return dbg; +} + +QDebug& operator<<(QDebug &dbg, const Sink::Storage::Revision &rev) +{ + dbg << rev.toDisplayString(); + return dbg; +} + +QDebug& operator<<(QDebug &dbg, const Sink::Storage::Key &key) +{ + dbg << key.toDisplayString(); + return dbg; +} + namespace Sink { namespace Storage { static const char *s_internalPrefix = "__internal"; static const int s_internalPrefixSize = strlen(s_internalPrefix); static const int s_lengthOfUid = 38; +// RFC 4122 Section 4.1.2 says 128 bits -> 16 bytes +//static const int s_lengthOfUid = 16; DbLayout::DbLayout() { @@ -194,6 +216,7 @@ bool DataStore::isInternalKey(const QByteArray &key) return key.startsWith(s_internalPrefix); } +/* QByteArray DataStore::assembleKey(const QByteArray &key, qint64 revision) { Q_ASSERT(revision <= 9223372036854775807); @@ -201,15 +224,17 @@ QByteArray DataStore::assembleKey(const QByteArray &key, qint64 revision) return key + QByteArray::number(revision).rightJustified(19, '0', false); } -QByteArray DataStore::uidFromKey(const QByteArray &key) +//QByteArray DataStore::uidFromKey(const QByteArray &key) +Identifier DataStore::uidFromKey(const QByteArray &key) { - return key.mid(0, s_lengthOfUid); + return Identifier::fromByteArray(key.mid(0, s_lengthOfUid)); } qint64 DataStore::revisionFromKey(const QByteArray &key) { return key.mid(s_lengthOfUid + 1).toLongLong(); } +*/ QByteArray DataStore::generateUid() { diff --git a/common/synchronizer.cpp b/common/synchronizer.cpp index 41ab1e9..f19c08a 100644 --- a/common/synchronizer.cpp +++ b/common/synchronizer.cpp @@ -616,7 +616,8 @@ KAsync::Job Synchronizer::replay(const QByteArray &type, const QByteArray Q_ASSERT(mEntityStore->hasTransaction()); const auto operation = metadataBuffer ? metadataBuffer->operation() : Sink::Operation_Creation; - const auto uid = Sink::Storage::DataStore::uidFromKey(key); + //const auto uid = Sink::Storage::DataStore::uidFromKey(key); + const auto uid = Sink::Storage::Key::fromDisplayByteArray(key).identifier().toDisplayByteArray(); const auto modifiedProperties = metadataBuffer->modifiedProperties() ? BufferUtils::fromVector(*metadataBuffer->modifiedProperties()) : QByteArrayList(); QByteArray oldRemoteId; diff --git a/common/typeindex.cpp b/common/typeindex.cpp index b18791f..0b78d59 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -25,8 +25,6 @@ #include #include -#include - using namespace Sink; static QByteArray getByteArray(const QVariant &value) @@ -53,13 +51,6 @@ static QByteArray getByteArray(const QVariant &value) return "toplevel"; } -template -static QByteArray padNumber(T number) -{ - static T uint_num_digits = (T)std::log10(std::numeric_limits::max()) + 1; - return QByteArray::number(number).rightJustified(uint_num_digits, '0'); -} - static QByteArray toSortableByteArrayImpl(const QDateTime &date) { // Sort invalid last diff --git a/common/utils.cpp b/common/utils.cpp index 3c54db4..e98365c 100644 --- a/common/utils.cpp +++ b/common/utils.cpp @@ -22,4 +22,5 @@ QByteArray Sink::createUuid() { return QUuid::createUuid().toByteArray(); + //return QUuid::createUuid().toRfc4122(); } diff --git a/common/utils.h b/common/utils.h index 253de61..7066d79 100644 --- a/common/utils.h +++ b/common/utils.h @@ -20,6 +20,17 @@ #include +#include + namespace Sink { - QByteArray createUuid(); + +QByteArray createUuid(); + +template +static QByteArray padNumber(T number) +{ + static T uint_num_digits = (T)std::log10(std::numeric_limits::max()) + 1; + return QByteArray::number(number).rightJustified(uint_num_digits, '0'); } + +} // namespace Sink -- cgit v1.2.3