From be09c96b977db014932a3d28b5ee6643ed5eff84 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Wed, 4 Jul 2018 16:12:58 +0200 Subject: Use key API in indexes --- common/index.cpp | 12 ++++++++ common/index.h | 3 ++ common/storage/entitystore.cpp | 17 ++++++----- common/typeindex.cpp | 66 +++++++++++++++++++++++++++--------------- common/typeindex.h | 17 ++++++----- 5 files changed, 76 insertions(+), 39 deletions(-) diff --git a/common/index.cpp b/common/index.cpp index 86a2dd5..238a745 100644 --- a/common/index.cpp +++ b/common/index.cpp @@ -2,6 +2,8 @@ #include "log.h" +using Sink::Storage::Identifier; + Index::Index(const QString &storageRoot, const QString &dbName, const QString &indexName, Sink::Storage::DataStore::AccessMode mode) : mTransaction(Sink::Storage::DataStore(storageRoot, dbName, mode).createTransaction(mode)), mDb(mTransaction.openDatabase(indexName.toLatin1(), std::function(), true)), @@ -32,6 +34,11 @@ Index::Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &tran { } +void Index::add(const Identifier &key, const QByteArray &value) +{ + add(key.toInternalByteArray(), value); +} + void Index::add(const QByteArray &key, const QByteArray &value) { Q_ASSERT(!key.isEmpty()); @@ -40,6 +47,11 @@ void Index::add(const QByteArray &key, const QByteArray &value) }); } +void Index::remove(const Identifier &key, const QByteArray &value) +{ + remove(key.toInternalByteArray(), value); +} + void Index::remove(const QByteArray &key, const QByteArray &value) { mDb.remove(key, value, [&] (const Sink::Storage::DataStore::Error &error) { diff --git a/common/index.h b/common/index.h index 492319e..833701e 100644 --- a/common/index.h +++ b/common/index.h @@ -6,6 +6,7 @@ #include #include "storage.h" #include "log.h" +#include "storage/key.h" /** * An index for value pairs. @@ -35,7 +36,9 @@ public: Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &); void add(const QByteArray &key, const QByteArray &value); + void add(const Sink::Storage::Identifier &key, const QByteArray &value); void remove(const QByteArray &key, const QByteArray &value); + void remove(const Sink::Storage::Identifier &key, const QByteArray &value); void lookup(const QByteArray &key, const std::function &resultHandler, const std::function &errorHandler, bool matchSubStringKeys = false); diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp index 3addf94..a1f6108 100644 --- a/common/storage/entitystore.cpp +++ b/common/storage/entitystore.cpp @@ -220,7 +220,9 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool SinkTraceCtx(d->logCtx) << "New entity " << entity; - d->typeIndex(type).add(entity.identifier(), entity, d->transaction, d->resourceContext.instanceId()); + const auto identifier = Identifier::fromDisplayByteArray(entity.identifier()); + + d->typeIndex(type).add(identifier, entity, d->transaction, d->resourceContext.instanceId()); //The maxRevision may have changed meanwhile if the entity created sub-entities const qint64 newRevision = maxRevision() + 1; @@ -237,7 +239,7 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool flatbuffers::FlatBufferBuilder fbb; d->resourceContext.adaptorFactory(type).createBuffer(entity, fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize()); - const auto key = Key(Identifier::fromDisplayByteArray(entity.identifier()), newRevision); + const auto key = Key(identifier, newRevision); DataStore::mainDatabase(d->transaction, type) .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), @@ -289,7 +291,8 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu { SinkTraceCtx(d->logCtx) << "Modified entity: " << newEntity; - d->typeIndex(type).modify(newEntity.identifier(), current, newEntity, d->transaction, d->resourceContext.instanceId()); + const auto identifier = Identifier::fromDisplayByteArray(newEntity.identifier()); + d->typeIndex(type).modify(identifier, current, newEntity, d->transaction, d->resourceContext.instanceId()); const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1; @@ -313,7 +316,7 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu flatbuffers::FlatBufferBuilder fbb; d->resourceContext.adaptorFactory(type).createBuffer(newEntity, fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize()); - const auto key = Key(Identifier::fromDisplayByteArray(newEntity.identifier()), newRevision); + const auto key = Key(identifier, newRevision); DataStore::mainDatabase(d->transaction, type) .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), @@ -331,8 +334,8 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu SinkWarningCtx(d->logCtx) << "Remove: Entity is already removed " << uid; return false; } - - d->typeIndex(type).remove(current.identifier(), current, d->transaction, d->resourceContext.instanceId()); + const auto identifier = Identifier::fromDisplayByteArray(uid); + d->typeIndex(type).remove(identifier, current, d->transaction, d->resourceContext.instanceId()); SinkTraceCtx(d->logCtx) << "Removed entity " << current; @@ -350,7 +353,7 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu flatbuffers::FlatBufferBuilder fbb; EntityBuffer::assembleEntityBuffer(fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize(), 0, 0, 0, 0); - const auto key = Key(Identifier::fromDisplayByteArray(uid), newRevision); + const auto key = Key(identifier, newRevision); DataStore::mainDatabase(d->transaction, type) .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb), diff --git a/common/typeindex.cpp b/common/typeindex.cpp index 0b78d59..887a146 100644 --- a/common/typeindex.cpp +++ b/common/typeindex.cpp @@ -27,6 +27,8 @@ using namespace Sink; +using Storage::Identifier; + static QByteArray getByteArray(const QVariant &value) { if (value.type() == QVariant::DateTime) { @@ -126,8 +128,8 @@ static void update(TypeIndex::Action action, const QByteArray &indexName, const void TypeIndex::addProperty(const QByteArray &property) { - auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { - update(action, indexName(property), getByteArray(value), identifier, transaction); + auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { + update(action, indexName(property), getByteArray(value), identifier.toInternalByteArray(), transaction); }; mIndexer.insert(property, indexer); mProperties << property; @@ -136,9 +138,9 @@ void TypeIndex::addProperty(const QByteArray &property) template <> void TypeIndex::addSortedProperty(const QByteArray &property) { - auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &value, + auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) { - update(action, sortedIndexName(property), toSortableByteArray(value), identifier, transaction); + update(action, sortedIndexName(property), toSortableByteArray(value), identifier.toInternalByteArray(), transaction); }; mSortIndexer.insert(property, indexer); mSortedProperties << property; @@ -147,10 +149,10 @@ void TypeIndex::addSortedProperty(const QByteArray &property) template <> void TypeIndex::addPropertyWithSorting(const QByteArray &property, const QByteArray &sortProperty) { - auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { + auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) { const auto date = sortValue.toDateTime(); const auto propertyValue = getByteArray(value); - update(action, indexName(property, sortProperty), propertyValue + toSortableByteArray(date), identifier, transaction); + update(action, indexName(property, sortProperty), propertyValue + toSortableByteArray(date), identifier.toInternalByteArray(), transaction); }; mGroupedSortIndexer.insert(property + sortProperty, indexer); mGroupedSortedProperties.insert(property, sortProperty); @@ -166,7 +168,7 @@ template <> void TypeIndex::addSampledPeriodIndex( const QByteArray &beginProperty, const QByteArray &endProperty) { - auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &begin, + auto indexer = [=](Action action, const Identifier &identifier, const QVariant &begin, const QVariant &end, Sink::Storage::DataStore::Transaction &transaction) { SinkTraceCtx(mLogCtx) << "Adding entity to sampled period index"; const auto beginDate = begin.toDateTime(); @@ -185,10 +187,10 @@ void TypeIndex::addSampledPeriodIndex( QByteArray bucketKey = padNumber(bucket); switch (action) { case TypeIndex::Add: - index.add(bucketKey, identifier); + index.add(bucketKey, identifier.toInternalByteArray()); break; case TypeIndex::Remove: - index.remove(bucketKey, identifier); + index.remove(bucketKey, identifier.toInternalByteArray()); break; } } @@ -198,7 +200,7 @@ void TypeIndex::addSampledPeriodIndex( mSampledPeriodIndexer.insert({ beginProperty, endProperty }, indexer); } -void TypeIndex::updateIndex(Action action, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) +void TypeIndex::updateIndex(Action action, const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { for (const auto &property : mProperties) { const auto value = entity.getProperty(property); @@ -239,7 +241,7 @@ void TypeIndex::abortTransaction() } } -void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) +void TypeIndex::add(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(Add, identifier, entity, transaction, resourceInstanceId); for (const auto &indexer : mCustomIndexer) { @@ -248,7 +250,7 @@ void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain: } } -void TypeIndex::modify(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) +void TypeIndex::modify(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(Remove, identifier, oldEntity, transaction, resourceInstanceId); updateIndex(Add, identifier, newEntity, transaction, resourceInstanceId); @@ -258,7 +260,7 @@ void TypeIndex::modify(const QByteArray &identifier, const Sink::ApplicationDoma } } -void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) +void TypeIndex::remove(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId) { updateIndex(Remove, identifier, entity, transaction, resourceInstanceId); for (const auto &indexer : mCustomIndexer) { @@ -275,7 +277,7 @@ static QVector indexLookup(Index &index, QueryBase::Comparator filte if (filter.comparator == Query::Comparator::Equals) { lookupKeys << valueToKey(filter.value); } else if (filter.comparator == Query::Comparator::In) { - for(const QVariant &value : filter.value.value()) { + for (const QVariant &value : filter.value.value()) { lookupKeys << valueToKey(value); } } else { @@ -283,7 +285,10 @@ static QVector indexLookup(Index &index, QueryBase::Comparator filte } for (const auto &lookupKey : lookupKeys) { - index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; }, + index.lookup(lookupKey, + [&](const QByteArray &value) { + keys << Identifier::fromInternalByteArray(value).toDisplayByteArray(); + }, [lookupKey](const Index::Error &error) { SinkWarning() << "Lookup error in index: " << error.message << lookupKey; }, @@ -315,7 +320,10 @@ static QVector sortedIndexLookup(Index &index, QueryBase::Comparator upperBound = bounds[1].toByteArray(); } - index.rangeLookup(lowerBound, upperBound, [&](const QByteArray &value) { keys << value; }, + index.rangeLookup(lowerBound, upperBound, + [&](const QByteArray &value) { + keys << Identifier::fromInternalByteArray(value).toDisplayByteArray(); + }, [bounds](const Index::Error &error) { SinkWarning() << "Lookup error in index:" << error.message << "with bounds:" << bounds[0] << bounds[1]; @@ -345,7 +353,7 @@ static QVector sampledIndexLookup(Index &index, QueryBase::Comparato index.rangeLookup(lowerBucket, upperBucket, [&](const QByteArray &value) { - keys << value.data(); + keys << Identifier::fromInternalByteArray(value).toDisplayByteArray(); }, [bounds](const Index::Error &error) { SinkWarning() << "Lookup error in index:" << error.message @@ -419,28 +427,38 @@ QVector TypeIndex::query(const Sink::QueryBase &query, QSet TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) +QVector TypeIndex::lookup(const QByteArray &property, const QVariant &value, + Sink::Storage::DataStore::Transaction &transaction) { SinkTraceCtx(mLogCtx) << "Index lookup on property: " << property << mSecondaryProperties.keys() << mProperties; if (mProperties.contains(property)) { QVector keys; Index index(indexName(property), transaction); const auto lookupKey = getByteArray(value); - index.lookup( - lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); + index.lookup(lookupKey, + [&](const QByteArray &value) { + keys << Identifier::fromInternalByteArray(value).toDisplayByteArray(); + }, + [property](const Index::Error &error) { + SinkWarning() << "Error in index: " << error.message << property; + }); SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys."; return keys; } else if (mSecondaryProperties.contains(property)) { - //Lookups on secondary indexes first lookup the key, and then lookup the results again to resolve to entity id's + // Lookups on secondary indexes first lookup the key, and then lookup the results again to + // resolve to entity id's QVector keys; auto resultProperty = mSecondaryProperties.value(property); QVector secondaryKeys; Index index(indexName(property + resultProperty), transaction); const auto lookupKey = getByteArray(value); - index.lookup( - lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; }); - SinkTraceCtx(mLogCtx) << "Looked up secondary keys for the following lookup key: " << lookupKey << " => " << secondaryKeys; + index.lookup(lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, + [property](const Index::Error &error) { + SinkWarning() << "Error in index: " << error.message << property; + }); + SinkTraceCtx(mLogCtx) << "Looked up secondary keys for the following lookup key: " << lookupKey + << " => " << secondaryKeys; for (const auto &secondary : secondaryKeys) { keys += lookup(resultProperty, secondary, transaction); } diff --git a/common/typeindex.h b/common/typeindex.h index 4e5a555..a701e9c 100644 --- a/common/typeindex.h +++ b/common/typeindex.h @@ -23,6 +23,7 @@ #include "query.h" #include "log.h" #include "indexer.h" +#include "storage/key.h" #include namespace Sink { @@ -89,9 +90,9 @@ public: addSampledPeriodIndex(Begin::name, End::name); } - void add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); - void modify(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); - void remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); + void add(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); + void modify(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); + void remove(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QVector query(const Sink::QueryBase &query, QSet &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QVector lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction); @@ -133,7 +134,7 @@ public: private: friend class Sink::Storage::EntityStore; - void updateIndex(Action action, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); + void updateIndex(Action action, const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId); QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const; QByteArray sortedIndexName(const QByteArray &property) const; QByteArray sampledPeriodIndexName(const QByteArray &rangeBeginProperty, const QByteArray &rangeEndProperty) const; @@ -147,8 +148,8 @@ private: QSet> mSampledPeriodProperties; QList mCustomIndexer; Sink::Storage::DataStore::Transaction *mTransaction; - QHash> mIndexer; - QHash> mSortIndexer; - QHash> mGroupedSortIndexer; - QHash, std::function> mSampledPeriodIndexer; + QHash> mIndexer; + QHash> mSortIndexer; + QHash> mGroupedSortIndexer; + QHash, std::function> mSampledPeriodIndexer; }; -- cgit v1.2.3