Use key API in indexes

5 files changed, 76 insertions, 39 deletions

diff --git a/common/index.cpp b/common/index.cpp
index 86a2dd5..238a745 100644
--- a/common/index.cpp
+++ b/common/index.cpp
@@ -2,6 +2,8 @@
 
 #include "log.h"
 
+using Sink::Storage::Identifier;
+
 Index::Index(const QString &storageRoot, const QString &dbName, const QString &indexName, Sink::Storage::DataStore::AccessMode mode)
     : mTransaction(Sink::Storage::DataStore(storageRoot, dbName, mode).createTransaction(mode)),
       mDb(mTransaction.openDatabase(indexName.toLatin1(), std::function<void(const Sink::Storage::DataStore::Error &)>(), true)),
@@ -32,6 +34,11 @@ Index::Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &tran
 {
 }
 
+void Index::add(const Identifier &key, const QByteArray &value)
+{
+    add(key.toInternalByteArray(), value);
+}
+
 void Index::add(const QByteArray &key, const QByteArray &value)
 {
     Q_ASSERT(!key.isEmpty());
@@ -40,6 +47,11 @@ void Index::add(const QByteArray &key, const QByteArray &value)
     });
 }
 
+void Index::remove(const Identifier &key, const QByteArray &value)
+{
+    remove(key.toInternalByteArray(), value);
+}
+
 void Index::remove(const QByteArray &key, const QByteArray &value)
 {
     mDb.remove(key, value, [&] (const Sink::Storage::DataStore::Error &error) {
diff --git a/common/index.h b/common/index.h
index 492319e..833701e 100644
--- a/common/index.h
+++ b/common/index.h
@@ -6,6 +6,7 @@
 #include <QString>
 #include "storage.h"
 #include "log.h"
+#include "storage/key.h"
 
 /**
  * An index for value pairs.
@@ -35,7 +36,9 @@ public:
     Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &);
 
     void add(const QByteArray &key, const QByteArray &value);
+    void add(const Sink::Storage::Identifier &key, const QByteArray &value);
     void remove(const QByteArray &key, const QByteArray &value);
+    void remove(const Sink::Storage::Identifier &key, const QByteArray &value);
 
     void lookup(const QByteArray &key, const std::function<void(const QByteArray &value)> &resultHandler, const std::function<void(const Error &error)> &errorHandler,
         bool matchSubStringKeys = false);
diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 3addf94..a1f6108 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -220,7 +220,9 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool
 
     SinkTraceCtx(d->logCtx) << "New entity " << entity;
 
-    d->typeIndex(type).add(entity.identifier(), entity, d->transaction, d->resourceContext.instanceId());
+    const auto identifier = Identifier::fromDisplayByteArray(entity.identifier());
+
+    d->typeIndex(type).add(identifier, entity, d->transaction, d->resourceContext.instanceId());
 
     //The maxRevision may have changed meanwhile if the entity created sub-entities
     const qint64 newRevision = maxRevision() + 1;
@@ -237,7 +239,7 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool
     flatbuffers::FlatBufferBuilder fbb;
     d->resourceContext.adaptorFactory(type).createBuffer(entity, fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize());
 
-    const auto key = Key(Identifier::fromDisplayByteArray(entity.identifier()), newRevision);
+    const auto key = Key(identifier, newRevision);
 
     DataStore::mainDatabase(d->transaction, type)
         .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb),
@@ -289,7 +291,8 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu
 {
     SinkTraceCtx(d->logCtx) << "Modified entity: " << newEntity;
 
-    d->typeIndex(type).modify(newEntity.identifier(), current, newEntity, d->transaction, d->resourceContext.instanceId());
+    const auto identifier = Identifier::fromDisplayByteArray(newEntity.identifier());
+    d->typeIndex(type).modify(identifier, current, newEntity, d->transaction, d->resourceContext.instanceId());
 
     const qint64 newRevision = DataStore::maxRevision(d->transaction) + 1;
 
@@ -313,7 +316,7 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu
     flatbuffers::FlatBufferBuilder fbb;
     d->resourceContext.adaptorFactory(type).createBuffer(newEntity, fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize());
 
-    const auto key = Key(Identifier::fromDisplayByteArray(newEntity.identifier()), newRevision);
+    const auto key = Key(identifier, newRevision);
 
     DataStore::mainDatabase(d->transaction, type)
         .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb),
@@ -331,8 +334,8 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu
         SinkWarningCtx(d->logCtx) << "Remove: Entity is already removed " << uid;
         return false;
     }
-
-    d->typeIndex(type).remove(current.identifier(), current, d->transaction, d->resourceContext.instanceId());
+    const auto identifier = Identifier::fromDisplayByteArray(uid);
+    d->typeIndex(type).remove(identifier, current, d->transaction, d->resourceContext.instanceId());
 
     SinkTraceCtx(d->logCtx) << "Removed entity " << current;
 
@@ -350,7 +353,7 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu
     flatbuffers::FlatBufferBuilder fbb;
     EntityBuffer::assembleEntityBuffer(fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize(), 0, 0, 0, 0);
 
-    const auto key = Key(Identifier::fromDisplayByteArray(uid), newRevision);
+    const auto key = Key(identifier, newRevision);
 
     DataStore::mainDatabase(d->transaction, type)
         .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb),
diff --git a/common/typeindex.cpp b/common/typeindex.cpp
index 0b78d59..887a146 100644
--- a/common/typeindex.cpp
+++ b/common/typeindex.cpp
@@ -27,6 +27,8 @@
 
 using namespace Sink;
 
+using Storage::Identifier;
+
 static QByteArray getByteArray(const QVariant &value)
 {
     if (value.type() == QVariant::DateTime) {
@@ -126,8 +128,8 @@ static void update(TypeIndex::Action action, const QByteArray &indexName, const
 
 void TypeIndex::addProperty(const QByteArray &property)
 {
-    auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) {
-        update(action, indexName(property), getByteArray(value), identifier, transaction);
+    auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction) {
+        update(action, indexName(property), getByteArray(value), identifier.toInternalByteArray(), transaction);
     };
     mIndexer.insert(property, indexer);
     mProperties << property;
@@ -136,9 +138,9 @@ void TypeIndex::addProperty(const QByteArray &property)
 template <>
 void TypeIndex::addSortedProperty<QDateTime>(const QByteArray &property)
 {
-    auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &value,
+    auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value,
                        Sink::Storage::DataStore::Transaction &transaction) {
-        update(action, sortedIndexName(property), toSortableByteArray(value), identifier, transaction);
+        update(action, sortedIndexName(property), toSortableByteArray(value), identifier.toInternalByteArray(), transaction);
     };
     mSortIndexer.insert(property, indexer);
     mSortedProperties << property;
@@ -147,10 +149,10 @@ void TypeIndex::addSortedProperty<QDateTime>(const QByteArray &property)
 template <>
 void TypeIndex::addPropertyWithSorting<QByteArray, QDateTime>(const QByteArray &property, const QByteArray &sortProperty)
 {
-    auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) {
+    auto indexer = [=](Action action, const Identifier &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction) {
         const auto date = sortValue.toDateTime();
         const auto propertyValue = getByteArray(value);
-        update(action, indexName(property, sortProperty), propertyValue + toSortableByteArray(date), identifier, transaction);
+        update(action, indexName(property, sortProperty), propertyValue + toSortableByteArray(date), identifier.toInternalByteArray(), transaction);
     };
     mGroupedSortIndexer.insert(property + sortProperty, indexer);
     mGroupedSortedProperties.insert(property, sortProperty);
@@ -166,7 +168,7 @@ template <>
 void TypeIndex::addSampledPeriodIndex<QDateTime, QDateTime>(
     const QByteArray &beginProperty, const QByteArray &endProperty)
 {
-    auto indexer = [=](Action action, const QByteArray &identifier, const QVariant &begin,
+    auto indexer = [=](Action action, const Identifier &identifier, const QVariant &begin,
                        const QVariant &end, Sink::Storage::DataStore::Transaction &transaction) {
         SinkTraceCtx(mLogCtx) << "Adding entity to sampled period index";
         const auto beginDate = begin.toDateTime();
@@ -185,10 +187,10 @@ void TypeIndex::addSampledPeriodIndex<QDateTime, QDateTime>(
             QByteArray bucketKey = padNumber(bucket);
             switch (action) {
                 case TypeIndex::Add:
-                    index.add(bucketKey, identifier);
+                    index.add(bucketKey, identifier.toInternalByteArray());
                     break;
                 case TypeIndex::Remove:
-                    index.remove(bucketKey, identifier);
+                    index.remove(bucketKey, identifier.toInternalByteArray());
                     break;
             }
         }
@@ -198,7 +200,7 @@ void TypeIndex::addSampledPeriodIndex<QDateTime, QDateTime>(
     mSampledPeriodIndexer.insert({ beginProperty, endProperty }, indexer);
 }
 
-void TypeIndex::updateIndex(Action action, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+void TypeIndex::updateIndex(Action action, const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
     for (const auto &property : mProperties) {
         const auto value = entity.getProperty(property);
@@ -239,7 +241,7 @@ void TypeIndex::abortTransaction()
     }
 }
 
-void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+void TypeIndex::add(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
     updateIndex(Add, identifier, entity, transaction, resourceInstanceId);
     for (const auto &indexer : mCustomIndexer) {
@@ -248,7 +250,7 @@ void TypeIndex::add(const QByteArray &identifier, const Sink::ApplicationDomain:
     }
 }
 
-void TypeIndex::modify(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+void TypeIndex::modify(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
     updateIndex(Remove, identifier, oldEntity, transaction, resourceInstanceId);
     updateIndex(Add, identifier, newEntity, transaction, resourceInstanceId);
@@ -258,7 +260,7 @@ void TypeIndex::modify(const QByteArray &identifier, const Sink::ApplicationDoma
     }
 }
 
-void TypeIndex::remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+void TypeIndex::remove(const Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
     updateIndex(Remove, identifier, entity, transaction, resourceInstanceId);
     for (const auto &indexer : mCustomIndexer) {
@@ -275,7 +277,7 @@ static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filte
     if (filter.comparator == Query::Comparator::Equals) {
         lookupKeys << valueToKey(filter.value);
     } else if (filter.comparator == Query::Comparator::In) {
-        for(const QVariant &value : filter.value.value<QVariantList>()) {
+        for (const QVariant &value : filter.value.value<QVariantList>()) {
             lookupKeys << valueToKey(value);
         }
     } else {
@@ -283,7 +285,10 @@ static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filte
     }
 
     for (const auto &lookupKey : lookupKeys) {
-        index.lookup(lookupKey, [&](const QByteArray &value) { keys << value; },
+        index.lookup(lookupKey,
+            [&](const QByteArray &value) {
+                keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
+            },
             [lookupKey](const Index::Error &error) {
                 SinkWarning() << "Lookup error in index: " << error.message << lookupKey;
             },
@@ -315,7 +320,10 @@ static QVector<QByteArray> sortedIndexLookup(Index &index, QueryBase::Comparator
         upperBound = bounds[1].toByteArray();
     }
 
-    index.rangeLookup(lowerBound, upperBound, [&](const QByteArray &value) { keys << value; },
+    index.rangeLookup(lowerBound, upperBound,
+        [&](const QByteArray &value) {
+            keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
+        },
         [bounds](const Index::Error &error) {
             SinkWarning() << "Lookup error in index:" << error.message
                           << "with bounds:" << bounds[0] << bounds[1];
@@ -345,7 +353,7 @@ static QVector<QByteArray> sampledIndexLookup(Index &index, QueryBase::Comparato
 
     index.rangeLookup(lowerBucket, upperBucket,
         [&](const QByteArray &value) {
-            keys << value.data();
+            keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
         },
         [bounds](const Index::Error &error) {
             SinkWarning() << "Lookup error in index:" << error.message
@@ -419,28 +427,38 @@ QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArr
     return {};
 }
 
-QVector<QByteArray> TypeIndex::lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction)
+QVector<QByteArray> TypeIndex::lookup(const QByteArray &property, const QVariant &value,
+    Sink::Storage::DataStore::Transaction &transaction)
 {
     SinkTraceCtx(mLogCtx) << "Index lookup on property: " << property << mSecondaryProperties.keys() << mProperties;
     if (mProperties.contains(property)) {
         QVector<QByteArray> keys;
         Index index(indexName(property), transaction);
         const auto lookupKey = getByteArray(value);
-        index.lookup(
-            lookupKey, [&](const QByteArray &value) { keys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
+        index.lookup(lookupKey,
+            [&](const QByteArray &value) {
+                keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
+            },
+            [property](const Index::Error &error) {
+                SinkWarning() << "Error in index: " << error.message << property;
+            });
         SinkTraceCtx(mLogCtx) << "Index lookup on " << property << " found " << keys.size() << " keys.";
         return keys;
     } else if (mSecondaryProperties.contains(property)) {
-        //Lookups on secondary indexes first lookup the key, and then lookup the results again to resolve to entity id's
+        // Lookups on secondary indexes first lookup the key, and then lookup the results again to
+        // resolve to entity id's
         QVector<QByteArray> keys;
         auto resultProperty = mSecondaryProperties.value(property);
 
         QVector<QByteArray> secondaryKeys;
         Index index(indexName(property + resultProperty), transaction);
         const auto lookupKey = getByteArray(value);
-        index.lookup(
-            lookupKey, [&](const QByteArray &value) { secondaryKeys << value; }, [property](const Index::Error &error) { SinkWarning() << "Error in index: " << error.message << property; });
-        SinkTraceCtx(mLogCtx) << "Looked up secondary keys for the following lookup key: " << lookupKey << " => " << secondaryKeys;
+        index.lookup(lookupKey, [&](const QByteArray &value) { secondaryKeys << value; },
+            [property](const Index::Error &error) {
+                SinkWarning() << "Error in index: " << error.message << property;
+            });
+        SinkTraceCtx(mLogCtx) << "Looked up secondary keys for the following lookup key: " << lookupKey
+                              << " => " << secondaryKeys;
         for (const auto &secondary : secondaryKeys) {
             keys += lookup(resultProperty, secondary, transaction);
         }
diff --git a/common/typeindex.h b/common/typeindex.h
index 4e5a555..a701e9c 100644
--- a/common/typeindex.h
+++ b/common/typeindex.h
@@ -23,6 +23,7 @@
 #include "query.h"
 #include "log.h"
 #include "indexer.h"
+#include "storage/key.h"
 #include <QByteArray>
 
 namespace Sink {
@@ -89,9 +90,9 @@ public:
         addSampledPeriodIndex<typename Begin::Type, typename End::Type>(Begin::name, End::name);
     }
 
-    void add(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
-    void modify(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
-    void remove(const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
+    void add(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
+    void modify(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
+    void remove(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
 
     QVector<QByteArray> query(const Sink::QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
     QVector<QByteArray> lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction);
@@ -133,7 +134,7 @@ public:
 
 private:
     friend class Sink::Storage::EntityStore;
-    void updateIndex(Action action, const QByteArray &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
+    void updateIndex(Action action, const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
     QByteArray indexName(const QByteArray &property, const QByteArray &sortProperty = QByteArray()) const;
     QByteArray sortedIndexName(const QByteArray &property) const;
     QByteArray sampledPeriodIndexName(const QByteArray &rangeBeginProperty, const QByteArray &rangeEndProperty) const;
@@ -147,8 +148,8 @@ private:
     QSet<QPair<QByteArray, QByteArray>> mSampledPeriodProperties;
     QList<Sink::Indexer::Ptr> mCustomIndexer;
     Sink::Storage::DataStore::Transaction *mTransaction;
-    QHash<QByteArray, std::function<void(Action, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction)>> mIndexer;
-    QHash<QByteArray, std::function<void(Action, const QByteArray &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction)>> mSortIndexer;
-    QHash<QByteArray, std::function<void(Action, const QByteArray &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction)>> mGroupedSortIndexer;
-    QHash<QPair<QByteArray, QByteArray>, std::function<void(Action, const QByteArray &identifier, const QVariant &begin, const QVariant &end, Sink::Storage::DataStore::Transaction &transaction)>> mSampledPeriodIndexer;
+    QHash<QByteArray, std::function<void(Action, const Sink::Storage::Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction)>> mIndexer;
+    QHash<QByteArray, std::function<void(Action, const Sink::Storage::Identifier &identifier, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction)>> mSortIndexer;
+    QHash<QByteArray, std::function<void(Action, const Sink::Storage::Identifier &identifier, const QVariant &value, const QVariant &sortValue, Sink::Storage::DataStore::Transaction &transaction)>> mGroupedSortIndexer;
+    QHash<QPair<QByteArray, QByteArray>, std::function<void(Action, const Sink::Storage::Identifier &identifier, const QVariant &begin, const QVariant &end, Sink::Storage::DataStore::Transaction &transaction)>> mSampledPeriodIndexer;
 };