From 23e13c91e44e9d1fcbe2215f16d10117de4d0e84 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Tue, 21 Aug 2018 12:03:40 +0200 Subject: Separate UIDs and revisions --- common/changereplay.cpp | 7 +- common/domain/typeimplementations.cpp | 14 +-- common/domain/typeimplementations_p.h | 10 +-- common/index.cpp | 8 +- common/mail/threadindexer.cpp | 4 +- common/storage.h | 36 ++++++-- common/storage/entitystore.cpp | 106 +++++++++++++--------- common/storage/key.cpp | 5 ++ common/storage/key.h | 1 + common/storage_common.cpp | 83 +++++++++++++---- common/storage_lmdb.cpp | 99 ++++++++++++++++----- sinksh/syntax_modules/sink_inspect.cpp | 5 +- tests/pipelinetest.cpp | 29 +++--- tests/storagetest.cpp | 157 ++++++++++++++++++++++++--------- 14 files changed, 397 insertions(+), 167 deletions(-) diff --git a/common/changereplay.cpp b/common/changereplay.cpp index d7f46dc..96162b8 100644 --- a/common/changereplay.cpp +++ b/common/changereplay.cpp @@ -116,16 +116,15 @@ KAsync::Job ChangeReplay::replayNextRevision() } else { // TODO: should not use internal representations const auto key = Storage::Key(Storage::Identifier::fromDisplayByteArray(uid), revision); - const auto internalKey = key.toInternalByteArray(); const auto displayKey = key.toDisplayByteArray(); QByteArray entityBuffer; DataStore::mainDatabase(mMainStoreTransaction, type) - .scan(internalKey, - [&entityBuffer](const QByteArray &key, const QByteArray &value) -> bool { + .scan(revision, + [&entityBuffer](const size_t, const QByteArray &value) -> bool { entityBuffer = value; return false; }, - [this, key](const DataStore::Error &) { SinkErrorCtx(mLogCtx) << "Failed to read the entity buffer " << key; }); + [this, key](const DataStore::Error &e) { SinkErrorCtx(mLogCtx) << "Failed to read the entity buffer " << key << "error:" << e; }); if (entityBuffer.isEmpty()) { SinkErrorCtx(mLogCtx) << "Failed to replay change " << key; diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp index aedf889..f969072 100644 --- a/common/domain/typeimplementations.cpp +++ b/common/domain/typeimplementations.cpp @@ -89,7 +89,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Mail::name} + ".main", 0}}, MailIndexConfig::databases()); + return merge(QMap{{QByteArray{Mail::name} + ".main", Storage::IntegerKeys}}, MailIndexConfig::databases()); } void TypeImplementation::configure(IndexPropertyMapper &indexPropertyMapper) @@ -132,7 +132,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Folder::name} + ".main", 0}}, FolderIndexConfig::databases()); + return merge(QMap{{QByteArray{Folder::name} + ".main", Storage::IntegerKeys}}, FolderIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -157,7 +157,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Contact::name} + ".main", 0}}, ContactIndexConfig::databases()); + return merge(QMap{{QByteArray{Contact::name} + ".main", Storage::IntegerKeys}}, ContactIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -185,7 +185,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Addressbook::name} + ".main", 0}}, AddressbookIndexConfig::databases()); + return merge(QMap{{QByteArray{Addressbook::name} + ".main", Storage::IntegerKeys}}, AddressbookIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -207,7 +207,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Event::name} + ".main", 0}}, EventIndexConfig::databases()); + return merge(QMap{{QByteArray{Event::name} + ".main", Storage::IntegerKeys}}, EventIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -235,7 +235,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Todo::name} + ".main", 0}}, TodoIndexConfig::databases()); + return merge(QMap{{QByteArray{Todo::name} + ".main", Storage::IntegerKeys}}, TodoIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) @@ -266,7 +266,7 @@ void TypeImplementation::configure(TypeIndex &index) QMap TypeImplementation::typeDatabases() { - return merge(QMap{{QByteArray{Calendar::name} + ".main", 0}}, CalendarIndexConfig::databases()); + return merge(QMap{{QByteArray{Calendar::name} + ".main", Storage::IntegerKeys}}, CalendarIndexConfig::databases()); } void TypeImplementation::configure(PropertyMapper &propertyMapper) diff --git a/common/domain/typeimplementations_p.h b/common/domain/typeimplementations_p.h index 51af113..bfdea77 100644 --- a/common/domain/typeimplementations_p.h +++ b/common/domain/typeimplementations_p.h @@ -57,7 +57,7 @@ public: template static QMap databases() { - return {{QByteArray{EntityType::name} +".index." + Property::name, 1}}; + return {{QByteArray{EntityType::name} +".index." + Property::name, Sink::Storage::AllowDuplicates}}; } }; @@ -74,7 +74,7 @@ public: template static QMap databases() { - return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, 1}}; + return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, Sink::Storage::AllowDuplicates}}; } }; @@ -90,7 +90,7 @@ public: template static QMap databases() { - return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", 1}}; + return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", Sink::Storage::AllowDuplicates}}; } }; @@ -106,7 +106,7 @@ public: template static QMap databases() { - return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, 1}}; + return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, Sink::Storage::AllowDuplicates}}; } }; @@ -142,7 +142,7 @@ public: template static QMap databases() { - return {{QByteArray{EntityType::name} +".index." + RangeBeginProperty::name + ".range." + RangeEndProperty::name, 1}}; + return {{QByteArray{EntityType::name} +".index." + RangeBeginProperty::name + ".range." + RangeEndProperty::name, Sink::Storage::AllowDuplicates}}; } }; diff --git a/common/index.cpp b/common/index.cpp index 238a745..bf8fcfc 100644 --- a/common/index.cpp +++ b/common/index.cpp @@ -6,7 +6,7 @@ using Sink::Storage::Identifier; Index::Index(const QString &storageRoot, const QString &dbName, const QString &indexName, Sink::Storage::DataStore::AccessMode mode) : mTransaction(Sink::Storage::DataStore(storageRoot, dbName, mode).createTransaction(mode)), - mDb(mTransaction.openDatabase(indexName.toLatin1(), std::function(), true)), + mDb(mTransaction.openDatabase(indexName.toLatin1(), std::function(), Sink::Storage::AllowDuplicates)), mName(indexName), mLogCtx("index." + indexName.toLatin1()) { @@ -14,7 +14,7 @@ Index::Index(const QString &storageRoot, const QString &dbName, const QString &i Index::Index(const QString &storageRoot, const QString &name, Sink::Storage::DataStore::AccessMode mode) : mTransaction(Sink::Storage::DataStore(storageRoot, name, mode).createTransaction(mode)), - mDb(mTransaction.openDatabase(name.toLatin1(), std::function(), true)), + mDb(mTransaction.openDatabase(name.toLatin1(), std::function(), Sink::Storage::AllowDuplicates)), mName(name), mLogCtx("index." + name.toLatin1()) { @@ -22,14 +22,14 @@ Index::Index(const QString &storageRoot, const QString &name, Sink::Storage::Dat Index::Index(const QString &storageRoot, const Sink::Storage::DbLayout &layout, Sink::Storage::DataStore::AccessMode mode) : mTransaction(Sink::Storage::DataStore(storageRoot, layout, mode).createTransaction(mode)), - mDb(mTransaction.openDatabase(layout.name, std::function(), true)), + mDb(mTransaction.openDatabase(layout.name, std::function(), Sink::Storage::AllowDuplicates)), mName(layout.name), mLogCtx("index." + layout.name) { } Index::Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &transaction) - : mDb(transaction.openDatabase(name, std::function(), true)), mName(name), + : mDb(transaction.openDatabase(name, std::function(), Sink::Storage::AllowDuplicates)), mName(name), mLogCtx("index." + name) { } diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp index c1d1aa8..b9de266 100644 --- a/common/mail/threadindexer.cpp +++ b/common/mail/threadindexer.cpp @@ -118,7 +118,7 @@ void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entit QMap ThreadIndexer::databases() { - return {{"mail.index.messageIdthreadId", 1}, - {"mail.index.threadIdmessageId", 1}}; + return {{"mail.index.messageIdthreadId", Sink::Storage::AllowDuplicates}, + {"mail.index.threadIdmessageId", Sink::Storage::AllowDuplicates}}; } diff --git a/common/storage.h b/common/storage.h index 8904148..53fcf41 100644 --- a/common/storage.h +++ b/common/storage.h @@ -32,6 +32,11 @@ namespace Sink { namespace Storage { +extern int AllowDuplicates; +extern int IntegerKeys; +// Only useful with AllowDuplicates +extern int IntegerValues; + struct SINK_EXPORT DbLayout { typedef QMap Databases; DbLayout(); @@ -80,15 +85,24 @@ public: */ bool write(const QByteArray &key, const QByteArray &value, const std::function &errorHandler = std::function()); + // TODO: change resultHandlers and errorHandlers to take size_t instead + // of QByteArray for keys + bool write(const size_t key, const QByteArray &value, const std::function &errorHandler = std::function()); + /** * Remove a key */ void remove(const QByteArray &key, const std::function &errorHandler = std::function()); + + void remove(const size_t key, const std::function &errorHandler = std::function()); + /** * Remove a key-value pair */ void remove(const QByteArray &key, const QByteArray &value, const std::function &errorHandler = std::function()); + void remove(const size_t key, const QByteArray &value, const std::function &errorHandler = std::function()); + /** * Read values with a given key. * @@ -101,6 +115,9 @@ public: int scan(const QByteArray &key, const std::function &resultHandler, const std::function &errorHandler = std::function(), bool findSubstringKeys = false, bool skipInternalKeys = true) const; + int scan(const size_t key, const std::function &resultHandler, + const std::function &errorHandler = std::function(), bool skipInternalKeys = true) const; + /** * Finds the last value in a series matched by prefix. * @@ -119,6 +136,10 @@ public: const std::function &errorHandler = std::function()) const; + int findAllInRange(const size_t lowerBound, const size_t upperBound, + const std::function &resultHandler, + const std::function &errorHandler = {}) const; + /** * Returns true if the database contains the substring key. */ @@ -163,8 +184,9 @@ public: QList getDatabaseNames() const; - NamedDatabase openDatabase(const QByteArray &name = {"default"}, - const std::function &errorHandler = {}, bool allowDuplicates = false) const; + NamedDatabase openDatabase(const QByteArray &name = { "default" }, + const std::function &errorHandler = {}, + int flags = 0) const; Transaction(Transaction &&other); Transaction &operator=(Transaction &&other); @@ -224,10 +246,12 @@ public: static qint64 cleanedUpRevision(const Transaction &); static void setCleanedUpRevision(Transaction &, qint64 revision); - static QByteArray getUidFromRevision(const Transaction &, qint64 revision); - static QByteArray getTypeFromRevision(const Transaction &, qint64 revision); - static void recordRevision(Transaction &, qint64 revision, const QByteArray &uid, const QByteArray &type); - static void removeRevision(Transaction &, qint64 revision); + static QByteArray getUidFromRevision(const Transaction &, size_t revision); + static size_t getLatestRevisionFromUid(Transaction &, const QByteArray &uid); + static QList getRevisionsUntilFromUid(DataStore::Transaction &, const QByteArray &uid, size_t lastRevision); + static QByteArray getTypeFromRevision(const Transaction &, size_t revision); + static void recordRevision(Transaction &, size_t revision, const QByteArray &uid, const QByteArray &type); + static void removeRevision(Transaction &, size_t revision); static void recordUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type); static void removeUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type); static void getUids(const QByteArray &type, const Transaction &, const std::function &); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 276ee6a..0640f1c 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -38,8 +38,9 @@ using namespace Sink::Storage; static QMap baseDbs() { - return {{"revisionType", 0}, - {"revisions", 0}, + return {{"revisionType", Storage::IntegerKeys}, + {"revisions", Storage::IntegerKeys}, + {"uidsToRevisions", Storage::AllowDuplicates | Storage::IntegerValues}, {"uids", 0}, {"default", 0}, {"__flagtable", 0}}; @@ -242,12 +243,13 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool const auto key = Key(identifier, newRevision); DataStore::mainDatabase(d->transaction, type) - .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), + .write(newRevision, BufferUtils::extractBuffer(fbb), [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << entity.identifier() << newRevision; }); + DataStore::setMaxRevision(d->transaction, newRevision); DataStore::recordRevision(d->transaction, newRevision, entity.identifier(), type); DataStore::recordUid(d->transaction, entity.identifier(), type); - SinkTraceCtx(d->logCtx) << "Wrote entity: " << entity.identifier() << type << newRevision; + SinkTraceCtx(d->logCtx) << "Wrote entity: " << entity.identifier() << type << newRevision << "key:" << key.toInternalByteArray(); return true; } @@ -319,8 +321,9 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu const auto key = Key(identifier, newRevision); DataStore::mainDatabase(d->transaction, type) - .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), + .write(newRevision, BufferUtils::extractBuffer(fbb), [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << newEntity.identifier() << newRevision; }); + DataStore::setMaxRevision(d->transaction, newRevision); DataStore::recordRevision(d->transaction, newRevision, newEntity.identifier(), type); SinkTraceCtx(d->logCtx) << "Wrote modified entity: " << newEntity.identifier() << type << newRevision; @@ -356,8 +359,9 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu const auto key = Key(identifier, newRevision); DataStore::mainDatabase(d->transaction, type) - .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), + .write(newRevision, BufferUtils::extractBuffer(fbb), [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << uid << newRevision; }); + DataStore::setMaxRevision(d->transaction, newRevision); DataStore::recordRevision(d->transaction, newRevision, uid, type); DataStore::removeUid(d->transaction, uid, type); @@ -375,30 +379,33 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision) } SinkTraceCtx(d->logCtx) << "Cleaning up revision " << revision << uid << bufferType; const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); - DataStore::mainDatabase(d->transaction, bufferType) - .scan(internalUid, - [&](const QByteArray &key, const QByteArray &data) -> bool { - EntityBuffer buffer(const_cast(data.data()), data.size()); - if (!buffer.isValid()) { - SinkWarningCtx(d->logCtx) << "Read invalid buffer from disk"; - } else { - const auto metadata = flatbuffers::GetRoot(buffer.metadataBuffer()); - const qint64 rev = metadata->revision(); - const auto isRemoval = metadata->operation() == Operation_Removal; - // Remove old revisions, and the current if the entity has already been removed - if (rev < revision || isRemoval) { - DataStore::removeRevision(d->transaction, rev); - DataStore::mainDatabase(d->transaction, bufferType).remove(key); - } - //Don't cleanup more than specified - if (rev >= revision) { - return false; - } - } - return true; - }, - [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error while reading: " << error.message; }, true); + // Remove old revisions + const auto revisionsToRemove = DataStore::getRevisionsUntilFromUid(d->transaction, uid, revision); + + for (const auto &revisionToRemove : revisionsToRemove) { + DataStore::removeRevision(d->transaction, revisionToRemove); + DataStore::mainDatabase(d->transaction, bufferType).remove(revisionToRemove); + } + + // And remove the specified revision only if marked for removal + DataStore::mainDatabase(d->transaction, bufferType).scan(revision, [&](size_t, const QByteArray &data) { + EntityBuffer buffer(const_cast(data.data()), data.size()); + if (!buffer.isValid()) { + SinkWarningCtx(d->logCtx) << "Read invalid buffer from disk"; + return false; + } + + const auto metadata = flatbuffers::GetRoot(buffer.metadataBuffer()); + const qint64 rev = metadata->revision(); + if (metadata->operation() == Operation_Removal) { + DataStore::removeRevision(d->transaction, revision); + DataStore::mainDatabase(d->transaction, bufferType).remove(revision); + } + + return false; + }); + DataStore::setCleanedUpRevision(d->transaction, revision); } @@ -437,13 +444,23 @@ QVector EntityStore::fullScan(const QByteArray &type) QSet keys; DataStore::mainDatabase(d->getTransaction(), type) .scan(QByteArray(), - [&](const QByteArray &key, const QByteArray &value) -> bool { - const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier(); - if (keys.contains(uid)) { + [&](const QByteArray &key, const QByteArray &data) -> bool { + EntityBuffer buffer(const_cast(data.data()), data.size()); + if (!buffer.isValid()) { + SinkWarningCtx(d->logCtx) << "Read invalid buffer from disk"; + return true; + } + + size_t revision = *reinterpret_cast(key.constData()); + + const auto metadata = flatbuffers::GetRoot(buffer.metadataBuffer()); + const QByteArray uid = DataStore::getUidFromRevision(d->getTransaction(), revision); + const auto identifier = Sink::Storage::Identifier::fromDisplayByteArray(uid); + if (keys.contains(identifier)) { //Not something that should persist if the replay works, so we keep a message for now. SinkTraceCtx(d->logCtx) << "Multiple revisions for uid: " << Sink::Storage::Key::fromInternalByteArray(key) << ". This is normal if changereplay has not completed yet."; } - keys << uid; + keys << identifier; return true; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during fullScan query: " << error.message; }); @@ -492,12 +509,12 @@ void EntityStore::indexLookup(const QByteArray &type, const QByteArray &property void EntityStore::readLatest(const QByteArray &type, const Identifier &id, const std::function callback) { Q_ASSERT(d); - const auto internalKey = id.toInternalByteArray(); + const size_t revision = DataStore::getLatestRevisionFromUid(d->getTransaction(), id.toDisplayByteArray()); auto db = DataStore::mainDatabase(d->getTransaction(), type); - db.findLatest(internalKey, - [=](const QByteArray &key, const QByteArray &value) { - const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); - callback(uid, Sink::EntityBuffer(value.data(), value.size())); + db.scan(revision, + [=](size_t, const QByteArray &value) { + callback(id.toDisplayByteArray(), Sink::EntityBuffer(value.data(), value.size())); + return false; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << id; }); } @@ -546,9 +563,9 @@ void EntityStore::readEntity(const QByteArray &type, const QByteArray &displayKe { const auto key = Key::fromDisplayByteArray(displayKey); auto db = DataStore::mainDatabase(d->getTransaction(), type); - db.scan(key.toInternalByteArray(), - [=](const QByteArray &key, const QByteArray &value) -> bool { - const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray(); + db.scan(key.revision().toSizeT(), + [=](size_t rev, const QByteArray &value) -> bool { + const auto uid = DataStore::getUidFromRevision(d->transaction, rev); callback(uid, Sink::EntityBuffer(value.data(), value.size())); return false; }, @@ -652,10 +669,10 @@ bool EntityStore::exists(const QByteArray &type, const QByteArray &uid) { bool found = false; bool alreadyRemoved = false; - const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray(); + const size_t revision = DataStore::getLatestRevisionFromUid(d->getTransaction(), uid); DataStore::mainDatabase(d->transaction, type) - .findLatest(internalUid, - [&found, &alreadyRemoved](const QByteArray &key, const QByteArray &data) { + .scan(revision, + [&found, &alreadyRemoved](size_t, const QByteArray &data) { auto entity = GetEntity(data.data()); if (entity && entity->metadata()) { auto metadata = GetMetadata(entity->metadata()->Data()); @@ -664,6 +681,7 @@ bool EntityStore::exists(const QByteArray &type, const QByteArray &uid) alreadyRemoved = true; } } + return true; }, [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read old revision from storage: " << error.message; }); if (!found) { diff --git a/common/storage/key.cpp b/common/storage/key.cpp index 2327061..a6567ea 100644 --- a/common/storage/key.cpp +++ b/common/storage/key.cpp @@ -155,6 +155,11 @@ qint64 Revision::toQint64() const return rev; } +size_t Revision::toSizeT() const +{ + return rev; +} + bool Revision::isValidInternal(const QByteArray &bytes) { if (bytes.size() != Revision::INTERNAL_REPR_SIZE) { diff --git a/common/storage/key.h b/common/storage/key.h index acd81cf..da90ddd 100644 --- a/common/storage/key.h +++ b/common/storage/key.h @@ -75,6 +75,7 @@ public: QByteArray toDisplayByteArray() const; static Revision fromDisplayByteArray(const QByteArray &bytes); qint64 toQint64() const; + size_t toSizeT() const; static bool isValidInternal(const QByteArray &); static bool isValidDisplay(const QByteArray &); diff --git a/common/storage_common.cpp b/common/storage_common.cpp index 264f223..ac246b2 100644 --- a/common/storage_common.cpp +++ b/common/storage_common.cpp @@ -117,26 +117,59 @@ qint64 DataStore::cleanedUpRevision(const DataStore::Transaction &transaction) return r; } -QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transaction, qint64 revision) +QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transaction, size_t revision) { QByteArray uid; - transaction.openDatabase("revisions") - .scan(QByteArray::number(revision), - [&](const QByteArray &, const QByteArray &value) -> bool { - uid = QByteArray{value.constData(), value.size()}; + transaction + .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys) + .scan(revision, + [&](const size_t, const QByteArray &value) -> bool { + uid = QByteArray{ value.constData(), value.size() }; return false; }, - [revision](const Error &error) { SinkWarning() << "Couldn't find uid for revision: " << revision << error.message; }); + [revision](const Error &error) { + SinkWarning() << "Couldn't find uid for revision: " << revision << error.message; + }); Q_ASSERT(!uid.isEmpty()); return uid; } -QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transaction, qint64 revision) +#include + +size_t DataStore::getLatestRevisionFromUid(DataStore::Transaction &t, const QByteArray &uid) +{ + size_t revision; + t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues) + .findLatest(uid, [&revision](const QByteArray &key, const QByteArray &value) { + revision = *reinterpret_cast(value.constData()); + }); + + return revision; +} + +QList DataStore::getRevisionsUntilFromUid(DataStore::Transaction &t, const QByteArray &uid, size_t lastRevision) +{ + QList queriedRevisions; + t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues) + .scan(uid, [&queriedRevisions, lastRevision](const QByteArray &, const QByteArray &value) { + size_t currentRevision = *reinterpret_cast(value.constData()); + if (currentRevision < lastRevision) { + queriedRevisions << currentRevision; + return true; + } + + return false; + }); + + return queriedRevisions; +} + +QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transaction, size_t revision) { QByteArray type; - transaction.openDatabase("revisionType") - .scan(QByteArray::number(revision), - [&](const QByteArray &, const QByteArray &value) -> bool { + transaction.openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys) + .scan(revision, + [&](const size_t, const QByteArray &value) -> bool { type = QByteArray{value.constData(), value.size()}; return false; }, @@ -145,17 +178,31 @@ QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transact return type; } -void DataStore::recordRevision(DataStore::Transaction &transaction, qint64 revision, const QByteArray &uid, const QByteArray &type) +void DataStore::recordRevision(DataStore::Transaction &transaction, size_t revision, + const QByteArray &uid, const QByteArray &type) { - // TODO use integerkeys - transaction.openDatabase("revisions").write(QByteArray::number(revision), uid); - transaction.openDatabase("revisionType").write(QByteArray::number(revision), type); + transaction + .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys) + .write(revision, uid); + transaction.openDatabase("uidsToRevisions", /* errorHandler = */ {}, AllowDuplicates | IntegerValues) + .write(uid, QByteArray::fromRawData(reinterpret_cast(&revision), sizeof(revision))); + transaction + .openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys) + .write(revision, type); } -void DataStore::removeRevision(DataStore::Transaction &transaction, qint64 revision) +void DataStore::removeRevision(DataStore::Transaction &transaction, size_t revision) { - transaction.openDatabase("revisions").remove(QByteArray::number(revision)); - transaction.openDatabase("revisionType").remove(QByteArray::number(revision)); + const QByteArray uid = getUidFromRevision(transaction, revision); + + transaction + .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys) + .remove(revision); + transaction.openDatabase("uidsToRevisions", /* errorHandler = */ {}, AllowDuplicates | IntegerValues) + .remove(uid, QByteArray::fromRawData(reinterpret_cast(&revision), sizeof(revision))); + transaction + .openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys) + .remove(revision); } void DataStore::recordUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type) @@ -207,7 +254,7 @@ DataStore::NamedDatabase DataStore::mainDatabase(const DataStore::Transaction &t Q_ASSERT(false); return {}; } - return t.openDatabase(type + ".main"); + return t.openDatabase(type + ".main", /* errorHandler= */ {}, IntegerKeys); } bool DataStore::NamedDatabase::contains(const QByteArray &uid) diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp index a007405..e3377b2 100644 --- a/common/storage_lmdb.cpp +++ b/common/storage_lmdb.cpp @@ -48,6 +48,10 @@ static QMutex sCreateDbiLock; static QHash sEnvironments; static QHash sDbis; +int AllowDuplicates = MDB_DUPSORT; +int IntegerKeys = MDB_INTEGERKEY; +int IntegerValues = MDB_INTEGERDUP; + int getErrorCode(int e) { switch (e) { @@ -101,14 +105,8 @@ static QList getDatabaseNames(MDB_txn *transaction) * and we always need to commit the transaction ASAP * We can only ever enter from one point per process. */ -static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, bool allowDuplicates, MDB_dbi &dbi) +static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, int flags, MDB_dbi &dbi) { - - unsigned int flags = 0; - if (allowDuplicates) { - flags |= MDB_DUPSORT; - } - MDB_dbi flagtableDbi; if (const int rc = mdb_dbi_open(transaction, "__flagtable", readOnly ? 0 : MDB_CREATE, &flagtableDbi)) { if (!readOnly) { @@ -130,6 +128,10 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, } } + if (flags & IntegerValues && !(flags & AllowDuplicates)) { + SinkWarning() << "Opening a database with integer values, but not duplicate keys"; + } + if (const int rc = mdb_dbi_open(transaction, db.constData(), flags, &dbi)) { //Create the db if it is not existing already if (rc == MDB_NOTFOUND && !readOnly) { @@ -165,7 +167,7 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, //Store the flags without the create option const auto ba = QByteArray::number(flags); value.mv_data = const_cast(static_cast(ba.constData())); - value.mv_size = db.size(); + value.mv_size = ba.size(); if (const int rc = mdb_put(transaction, flagtableDbi, &key, &value, MDB_NOOVERWRITE)) { //We expect this to fail if we're only creating the dbi but not the db if (rc != MDB_KEYEXIST) { @@ -175,7 +177,7 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, } else { //It's not an error if we only want to read if (!readOnly) { - SinkWarning() << "Failed to open db " << QByteArray(mdb_strerror(rc)); + SinkWarning() << "Failed to open db " << db << "error:" << QByteArray(mdb_strerror(rc)); return true; } return false; @@ -187,8 +189,14 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, class DataStore::NamedDatabase::Private { public: - Private(const QByteArray &_db, bool _allowDuplicates, const std::function &_defaultErrorHandler, const QString &_name, MDB_txn *_txn) - : db(_db), transaction(_txn), allowDuplicates(_allowDuplicates), defaultErrorHandler(_defaultErrorHandler), name(_name) + Private(const QByteArray &_db, int _flags, + const std::function &_defaultErrorHandler, + const QString &_name, MDB_txn *_txn) + : db(_db), + transaction(_txn), + flags(_flags), + defaultErrorHandler(_defaultErrorHandler), + name(_name) { } @@ -199,7 +207,7 @@ public: QByteArray db; MDB_txn *transaction; MDB_dbi dbi; - bool allowDuplicates; + int flags; std::function defaultErrorHandler; QString name; bool createdNewDbi = false; @@ -313,7 +321,7 @@ public: } else { dbiTransaction = transaction; } - if (createDbi(dbiTransaction, db, readOnly, allowDuplicates, dbi)) { + if (createDbi(dbiTransaction, db, readOnly, flags, dbi)) { if (readOnly) { mdb_txn_commit(dbiTransaction); Q_ASSERT(!sDbis.contains(dbiName)); @@ -371,6 +379,13 @@ DataStore::NamedDatabase::~NamedDatabase() delete d; } +bool DataStore::NamedDatabase::write(const size_t key, const QByteArray &value, + const std::function &errorHandler) +{ + auto baKey = QByteArray::fromRawData(reinterpret_cast(&key), sizeof(key)); + return write(baKey, value, errorHandler); +} + bool DataStore::NamedDatabase::write(const QByteArray &sKey, const QByteArray &sValue, const std::function &errorHandler) { if (!d || !d->transaction) { @@ -407,11 +422,25 @@ bool DataStore::NamedDatabase::write(const QByteArray &sKey, const QByteArray &s return !rc; } +void DataStore::NamedDatabase::remove( + const size_t key, const std::function &errorHandler) +{ + auto baKey = QByteArray::fromRawData(reinterpret_cast(&key), sizeof(key)); + return remove(baKey, errorHandler); +} + void DataStore::NamedDatabase::remove(const QByteArray &k, const std::function &errorHandler) { remove(k, QByteArray(), errorHandler); } +void DataStore::NamedDatabase::remove(const size_t key, const QByteArray &value, + const std::function &errorHandler) +{ + auto baKey = QByteArray::fromRawData(reinterpret_cast(&key), sizeof(key)); + return remove(baKey, value, errorHandler); +} + void DataStore::NamedDatabase::remove(const QByteArray &k, const QByteArray &value, const std::function &errorHandler) { if (!d || !d->transaction) { @@ -445,6 +474,19 @@ void DataStore::NamedDatabase::remove(const QByteArray &k, const QByteArray &val } } +int DataStore::NamedDatabase::scan(const size_t key, + const std::function &resultHandler, + const std::function &errorHandler, bool skipInternalKeys) const +{ + auto baKey = QByteArray::fromRawData(reinterpret_cast(&key), sizeof(key)); + return scan(baKey, + [&resultHandler](const QByteArray &key, const QByteArray &value) { + size_t integerKey = *reinterpret_cast(key.constData()); + return resultHandler(integerKey, value); + }, + errorHandler, /* findSubstringKeys = */ false, skipInternalKeys); +} + int DataStore::NamedDatabase::scan(const QByteArray &k, const std::function &resultHandler, const std::function &errorHandler, bool findSubstringKeys, bool skipInternalKeys) const { @@ -471,8 +513,10 @@ int DataStore::NamedDatabase::scan(const QByteArray &k, const std::functionallowDuplicates || findSubstringKeys) { - MDB_cursor_op op = d->allowDuplicates ? MDB_SET : MDB_FIRST; + bool allowDuplicates = d->flags & AllowDuplicates; + + if (k.isEmpty() || allowDuplicates || findSubstringKeys) { + MDB_cursor_op op = allowDuplicates ? MDB_SET : MDB_FIRST; if (findSubstringKeys) { op = MDB_SET_RANGE; } @@ -490,7 +534,7 @@ int DataStore::NamedDatabase::scan(const QByteArray &k, const std::functionallowDuplicates && !findSubstringKeys) ? MDB_NEXT_DUP : MDB_NEXT; + MDB_cursor_op nextOp = (allowDuplicates && !findSubstringKeys) ? MDB_NEXT_DUP : MDB_NEXT; while ((rc = mdb_cursor_get(cursor, &key, &data, nextOp)) == 0) { const auto current = QByteArray::fromRawData((char *)key.mv_data, key.mv_size); // Every consequitive lookup simply iterates through the list @@ -602,6 +646,15 @@ void DataStore::NamedDatabase::findLatest(const QByteArray &k, const std::functi return; } +int DataStore::NamedDatabase::findAllInRange(const size_t lowerBound, const size_t upperBound, + const std::function &resultHandler, + const std::function &errorHandler) const +{ + auto baLowerBound = QByteArray::fromRawData(reinterpret_cast(&lowerBound), sizeof(size_t)); + auto baUpperBound = QByteArray::fromRawData(reinterpret_cast(&upperBound), sizeof(size_t)); + return findAllInRange(baLowerBound, baUpperBound, resultHandler, errorHandler); +} + int DataStore::NamedDatabase::findAllInRange(const QByteArray &lowerBound, const QByteArray &upperBound, const std::function &resultHandler, const std::function &errorHandler) const @@ -862,7 +915,8 @@ static bool ensureCorrectDb(DataStore::NamedDatabase &database, const QByteArray return !openedTheWrongDatabase; } -DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &db, const std::function &errorHandler, bool allowDuplicates) const +DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &db, + const std::function &errorHandler, int flags) const { if (!d) { SinkError() << "Tried to open database on invalid transaction: " << db; @@ -871,7 +925,8 @@ DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray & Q_ASSERT(d->transaction); // We don't now if anything changed d->implicitCommit = true; - auto p = new DataStore::NamedDatabase::Private(db, allowDuplicates, d->defaultErrorHandler, d->name, d->transaction); + auto p = new DataStore::NamedDatabase::Private( + db, flags, d->defaultErrorHandler, d->name, d->transaction); auto ret = p->openDatabase(d->requestedRead, errorHandler); if (!ret) { delete p; @@ -1049,11 +1104,11 @@ public: //Create dbis from the given layout. for (auto it = layout.tables.constBegin(); it != layout.tables.constEnd(); it++) { - const bool allowDuplicates = it.value(); + const int flags = it.value(); MDB_dbi dbi = 0; const auto db = it.key(); const auto dbiName = name + db; - if (createDbi(transaction, db, readOnly, allowDuplicates, dbi)) { + if (createDbi(transaction, db, readOnly, flags, dbi)) { sDbis.insert(dbiName, dbi); } } @@ -1063,8 +1118,8 @@ public: MDB_dbi dbi = 0; const auto dbiName = name + db; //We're going to load the flags anyways. - bool allowDuplicates = false; - if (createDbi(transaction, db, readOnly, allowDuplicates, dbi)) { + const int flags = 0; + if (createDbi(transaction, db, readOnly, flags, dbi)) { sDbis.insert(dbiName, dbi); } } diff --git a/sinksh/syntax_modules/sink_inspect.cpp b/sinksh/syntax_modules/sink_inspect.cpp index 1d2d90f..956431a 100644 --- a/sinksh/syntax_modules/sink_inspect.cpp +++ b/sinksh/syntax_modules/sink_inspect.cpp @@ -87,7 +87,7 @@ bool inspect(const QStringList &args, State &state) [&] (const Sink::Storage::DataStore::Error &e) { Q_ASSERT(false); state.printError(e.message); - }, false); + }, Sink::Storage::IntegerKeys); auto ridMap = syncTransaction.openDatabase("localid.mapping." + type, [&] (const Sink::Storage::DataStore::Error &e) { @@ -108,7 +108,8 @@ bool inspect(const QStringList &args, State &state) QSet uids; db.scan("", [&] (const QByteArray &key, const QByteArray &data) { - uids.insert(Key::fromInternalByteArray(key).identifier().toDisplayByteArray()); + size_t revision = *reinterpret_cast(key.constData()); + uids.insert(Sink::Storage::DataStore::getUidFromRevision(transaction, revision)); return true; }, [&](const Sink::Storage::DataStore::Error &e) { diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp index b41a5c2..47d443f 100644 --- a/tests/pipelinetest.cpp +++ b/tests/pipelinetest.cpp @@ -28,26 +28,29 @@ static void removeFromDisk(const QString &name) store.removeFromDisk(); } -static QList getKeys(const QByteArray &dbEnv, const QByteArray &name) +static QList getKeys(const QByteArray &dbEnv, const QByteArray &name) { Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db = transaction.openDatabase(name, nullptr, false); - QList result; + QList result; db.scan("", [&](const QByteArray &key, const QByteArray &value) { - result << key; + size_t revision = *reinterpret_cast(key.constData()); + result << Sink::Storage::Key(Sink::Storage::Identifier::fromDisplayByteArray( + Sink::Storage::DataStore::getUidFromRevision(transaction, revision)), + revision); return true; }); return result; } -static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const QByteArray &uid) +static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const Sink::Storage::Key &key) { Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db = transaction.openDatabase(name, nullptr, false); QByteArray result; - db.scan(uid, [&](const QByteArray &key, const QByteArray &value) { + db.scan(key.revision().toSizeT(), [&](size_t rev, const QByteArray &value) { result = value; return true; }); @@ -251,7 +254,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); // Execute the modification @@ -264,7 +267,7 @@ private slots: key.setRevision(2); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); @@ -299,7 +302,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); @@ -322,7 +325,7 @@ private slots: key.setRevision(3); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); @@ -343,7 +346,7 @@ private slots: auto result = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(result.size(), 1); - const auto uid = Sink::Storage::Key::fromInternalByteArray(result.first()).identifier().toDisplayByteArray(); + const auto uid = result.first().identifier().toDisplayByteArray(); // Delete entity auto deleteCommand = deleteEntityCommand(uid, 1); @@ -386,7 +389,7 @@ private slots: pipeline.startTransaction(); auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - const auto uid = Sink::Storage::Key::fromInternalByteArray(keys.first()).identifier().toDisplayByteArray(); + const auto uid = keys.first().identifier().toDisplayByteArray(); { auto modifyCommand = modifyEntityCommand(createEvent(entityFbb, "summary2"), uid, 1); pipeline.modifiedEntity(modifyCommand.constData(), modifyCommand.size()); @@ -427,7 +430,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); //Simulate local modification @@ -453,7 +456,7 @@ private slots: key.setRevision(3); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index 81acc13..39fd380 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp @@ -227,7 +227,7 @@ private slots: bool gotError = false; Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("default", nullptr, false); + auto db = transaction.openDatabase("default"); db.write("key", "value"); db.write("key", "value"); @@ -250,9 +250,10 @@ private slots: { bool gotResult = false; bool gotError = false; - Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0x04}}}, Sink::Storage::DataStore::ReadWrite); + const int flags = Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", flags}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("default", nullptr, true); + auto db = transaction.openDatabase("default", nullptr, flags); db.write("key", "value1"); db.write("key", "value2"); int numValues = db.scan("key", @@ -357,7 +358,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); store.createTransaction(Sink::Storage::DataStore::ReadWrite) - .openDatabase("test", nullptr, true) + .openDatabase("test", nullptr, Sink::Storage::AllowDuplicates) .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) { qDebug() << error.message; gotError = true; @@ -368,9 +369,10 @@ private slots: // By default we want only exact matches void testSubstringKeys() { - Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0x04}}}, Sink::Storage::DataStore::ReadWrite); + const int flags = Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", flags}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, flags); db.write("sub", "value1"); db.write("subsub", "value2"); int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); @@ -382,7 +384,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub", "value1"); db.write("subsub", "value2"); db.write("wubsub", "value3"); @@ -395,7 +397,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); db.write("sub", "value1"); db.write("subsub", "value2"); db.write("wubsub", "value3"); @@ -408,7 +410,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub_2", "value2"); db.write("sub_1", "value1"); db.write("sub_3", "value3"); @@ -429,7 +431,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); db.write("sub1", "value1"); int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); @@ -440,7 +442,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub1", "value1"); db.write("sub2", "value2"); db.write("wub3", "value3"); @@ -455,7 +457,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub2", "value2"); QByteArray result; db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; }); @@ -467,7 +469,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub2", "value2"); db.write("wub3", "value3"); QByteArray result; @@ -478,8 +480,8 @@ private slots: static QMap baseDbs() { - return {{"revisionType", 0}, - {"revisions", 0}, + return {{"revisionType", Sink::Storage::IntegerKeys}, + {"revisions", Sink::Storage::IntegerKeys}, {"uids", 0}, {"default", 0}, {"__flagtable", 0}}; @@ -499,7 +501,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); QByteArray result; - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); const auto uid = "{c5d06a9f-1534-4c52-b8ea-415db68bdadf}"; //Ensure we can sort 1 and 10 properly (by default string comparison 10 comes before 6) const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid); @@ -523,7 +525,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results; db.findAllInRange("0002", "0004", [&](const QByteArray &key, const QByteArray &value) { results << value; }); @@ -535,7 +537,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -559,7 +561,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -571,7 +573,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -601,21 +603,21 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false); + auto db = transaction.openDatabase("testTransactionVisibility"); db.write("key1", "foo"); QCOMPARE(readValue(db, "key1"), QByteArray("foo")); { auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db2 = transaction2 - .openDatabase("testTransactionVisibility", nullptr, false); + .openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray()); } transaction.commit(); { auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db2 = transaction2 - .openDatabase("testTransactionVisibility", nullptr, false); + .openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); } @@ -627,16 +629,16 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"a", 0}, {"b", 0}, {"c", 0}}}, Sink::Storage::DataStore::ReadWrite); { auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - transaction.openDatabase("a", nullptr, false); - transaction.openDatabase("b", nullptr, false); - transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("a"); + transaction.openDatabase("b"); + transaction.openDatabase("c"); transaction.commit(); } auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); for (int i = 0; i < 1000; i++) { - transaction.openDatabase("a", nullptr, false); - transaction.openDatabase("b", nullptr, false); - transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("a"); + transaction.openDatabase("b"); + transaction.openDatabase("c"); transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); } } @@ -662,11 +664,11 @@ private slots: // Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly); // auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); // for (int i = 0; i < 100000; i++) { - // transaction.openDatabase("a", nullptr, false); - // transaction.openDatabase("b", nullptr, false); - // transaction.openDatabase("c", nullptr, false); - // transaction.openDatabase("p", nullptr, false); - // transaction.openDatabase("q", nullptr, false); + // transaction.openDatabase("a"); + // transaction.openDatabase("b"); + // transaction.openDatabase("c"); + // transaction.openDatabase("p"); + // transaction.openDatabase("q"); // } // }); // } @@ -733,7 +735,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false); + auto db = transaction.openDatabase("testTransactionVisibility"); db.write("key1", "foo"); QCOMPARE(readValue(db, "key1"), QByteArray("foo")); transaction.commit(); @@ -748,12 +750,12 @@ private slots: //This transaction should open the dbi auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false); + auto db2 = transaction2.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); //This transaction should have the dbi available auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false); + auto db3 = transaction3.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); } @@ -766,20 +768,95 @@ private slots: //This transaction should open the dbi auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false); + auto db2 = transaction2.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); //This transaction should have the dbi available (creating two write transactions obviously doesn't work) //NOTE: we don't support this scenario. A write transaction must commit or abort before a read transaction opens the same database. // auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - // auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false); + // auto db3 = transaction3.openDatabase("testTransactionVisibility"); // QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); //Ensure we can still open further dbis in the write transaction - auto db4 = transaction2.openDatabase("anotherDb", nullptr, false); + auto db4 = transaction2.openDatabase("anotherDb"); } } + + void testIntegerKeys() + { + const int flags = Sink::Storage::IntegerKeys; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "test", flags } } }, Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testIntegerKeys", {}, flags); + db.write(0, "value1"); + db.write(1, "value2"); + QByteArray result; + int numValues = db.scan(0, [&](size_t, const QByteArray &value) -> bool { + result = value; + return true; + }); + + QCOMPARE(numValues, 1); + QCOMPARE(result, "value1"); + } + + void testDuplicateIntegerKeys() + { + const int flags = Sink::Storage::IntegerKeys | Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "testDuplicateIntegerKeys", flags} } }, + Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags); + db.write(0, "value1"); + db.write(1, "value2"); + db.write(1, "value3"); + QSet results; + int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { + results << value; + return true; + }); + + QCOMPARE(numValues, 2); + QCOMPARE(results.size(), 2); + QVERIFY(results.contains("value2")); + QVERIFY(results.contains("value3")); + } + + void testDuplicateWithIntegerValues() + { + const int flags = Sink::Storage::AllowDuplicates | Sink::Storage::IntegerValues; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "testDuplicateWithIntegerValues", flags} } }, + Sink::Storage::DataStore::ReadWrite); + + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags); + + const size_t number1 = 1; + const size_t number2 = 2; + + const QByteArray number1BA = QByteArray::fromRawData(reinterpret_cast(&number1), sizeof(size_t)); + const QByteArray number2BA = QByteArray::fromRawData(reinterpret_cast(&number2), sizeof(size_t)); + + db.write(0, number1BA); + db.write(1, number2BA); + db.write(1, number1BA); + + QList results; + int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { + results << value; + return true; + }); + + QCOMPARE(numValues, 2); + QCOMPARE(results.size(), 2); + QVERIFY(results[0] == number1BA); + QVERIFY(results[1] == number2BA); + } + }; QTEST_MAIN(StorageTest) -- cgit v1.2.3