Avoid unnecessary Identifier conversions in performance ciritical code.

This fixes the performance regressions to a state where we are roughly
at the same performance as pre Identifier (but not any better either).

7 files changed, 40 insertions, 27 deletions

diff --git a/common/datastorequery.cpp b/common/datastorequery.cpp
index 52243da..43b4660 100644
--- a/common/datastorequery.cpp
+++ b/common/datastorequery.cpp
@@ -46,14 +46,10 @@ class Source : public FilterBase {
     QVector<Identifier> mIncrementalIds;
     QVector<Identifier>::ConstIterator mIncrementalIt;
 
-    Source (const QVector<QByteArray> &ids, DataStoreQuery *store)
+    Source (const QVector<Identifier> &ids, DataStoreQuery *store)
         : FilterBase(store),
-        mIds()
+        mIds(ids)
     {
-        mIds.reserve(ids.size());
-        for (const auto &id : ids) {
-            mIds.append(Identifier::fromDisplayByteArray(id));
-        }
         mIt = mIds.constBegin();
     }
 
@@ -614,7 +610,11 @@ void DataStoreQuery::setupQuery(const Sink::QueryBase &query_)
     mSource = [&]() {
         if (!query.ids().isEmpty()) {
             //We have a set of ids as a starting point
-            return Source::Ptr::create(query.ids().toVector(), this);
+            QVector<Identifier> ids;
+            for (const auto & id: query.ids()) {
+                ids.append(Identifier::fromDisplayByteArray(id));
+            }
+            return Source::Ptr::create(ids, this);
         } else {
             QSet<QByteArrayList> appliedFilters;
             auto resultSet = mStore.indexLookup(mType, query, appliedFilters, appliedSorting);
diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 71690fe..1ba7c6c 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -426,19 +426,19 @@ bool EntityStore::cleanupRevisions(qint64 revision)
     return cleanupIsNecessary;
 }
 
-QVector<QByteArray> EntityStore::fullScan(const QByteArray &type)
+QVector<Identifier> EntityStore::fullScan(const QByteArray &type)
 {
     SinkTraceCtx(d->logCtx) << "Looking for : " << type;
     if (!d->exists()) {
         SinkTraceCtx(d->logCtx) << "Database is not existing: " << type;
-        return QVector<QByteArray>();
+        return {};
     }
     //The scan can return duplicate results if we have multiple revisions, so we use a set to deduplicate.
-    QSet<QByteArray> keys;
+    QSet<Identifier> keys;
     DataStore::mainDatabase(d->getTransaction(), type)
         .scan(QByteArray(),
             [&](const QByteArray &key, const QByteArray &value) -> bool {
-                const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray();
+                const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier();
                 if (keys.contains(uid)) {
                     //Not something that should persist if the replay works, so we keep a message for now.
                     SinkTraceCtx(d->logCtx) << "Multiple revisions for key: " << key;
@@ -452,11 +452,11 @@ QVector<QByteArray> EntityStore::fullScan(const QByteArray &type)
     return keys.toList().toVector();
 }
 
-QVector<QByteArray> EntityStore::indexLookup(const QByteArray &type, const QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting)
+QVector<Identifier> EntityStore::indexLookup(const QByteArray &type, const QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting)
 {
     if (!d->exists()) {
         SinkTraceCtx(d->logCtx) << "Database is not existing: " << type;
-        return QVector<QByteArray>();
+        return {};
     }
     return d->typeIndex(type).query(query, appliedFilters, appliedSorting, d->getTransaction(), d->resourceContext.instanceId());
 }
diff --git a/common/storage/entitystore.h b/common/storage/entitystore.h
index 619b884..7979798 100644
--- a/common/storage/entitystore.h
+++ b/common/storage/entitystore.h
@@ -57,8 +57,8 @@ public:
     void abortTransaction();
     bool hasTransaction() const;
 
-    QVector<QByteArray> fullScan(const QByteArray &type);
-    QVector<QByteArray> indexLookup(const QByteArray &type, const QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting);
+    QVector<Sink::Storage::Identifier> fullScan(const QByteArray &type);
+    QVector<Sink::Storage::Identifier> indexLookup(const QByteArray &type, const QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting);
     QVector<Sink::Storage::Identifier> indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value);
     void indexLookup(const QByteArray &type, const QByteArray &property, const QVariant &value, const std::function<void(const QByteArray &uid)> &callback);
     template<typename EntityType, typename PropertyType>
diff --git a/common/storage/key.cpp b/common/storage/key.cpp
index 23d7a6a..2327061 100644
--- a/common/storage/key.cpp
+++ b/common/storage/key.cpp
@@ -26,6 +26,12 @@ using Sink::Storage::Identifier;
 using Sink::Storage::Key;
 using Sink::Storage::Revision;
 
+
+uint Sink::Storage::qHash(const Sink::Storage::Identifier &identifier)
+{
+    return qHash(identifier.toInternalByteArray());
+}
+
 QDebug &operator<<(QDebug &dbg, const Identifier &id)
 {
     dbg << id.toDisplayString();
diff --git a/common/storage/key.h b/common/storage/key.h
index 211aea7..baabe38 100644
--- a/common/storage/key.h
+++ b/common/storage/key.h
@@ -119,6 +119,8 @@ private:
     Revision rev;
 };
 
+SINK_EXPORT uint qHash(const Sink::Storage::Identifier &);
+
 } // namespace Storage
 } // namespace Sink
 
diff --git a/common/typeindex.cpp b/common/typeindex.cpp
index 47512e8..646a60f 100644
--- a/common/typeindex.cpp
+++ b/common/typeindex.cpp
@@ -265,10 +265,10 @@ void TypeIndex::remove(const Identifier &identifier, const Sink::ApplicationDoma
     }
 }
 
-static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filter,
+static QVector<Identifier> indexLookup(Index &index, QueryBase::Comparator filter,
     std::function<QByteArray(const QVariant &)> valueToKey = getByteArray)
 {
-    QVector<QByteArray> keys;
+    QVector<Identifier> keys;
     QByteArrayList lookupKeys;
     if (filter.comparator == Query::Comparator::Equals) {
         lookupKeys << valueToKey(filter.value);
@@ -283,7 +283,7 @@ static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filte
     for (const auto &lookupKey : lookupKeys) {
         index.lookup(lookupKey,
             [&](const QByteArray &value) {
-                keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
+                keys << Identifier::fromInternalByteArray(value);
             },
             [lookupKey](const Index::Error &error) {
                 SinkWarning() << "Lookup error in index: " << error.message << lookupKey;
@@ -293,7 +293,7 @@ static QVector<QByteArray> indexLookup(Index &index, QueryBase::Comparator filte
     return keys;
 }
 
-static QVector<QByteArray> sortedIndexLookup(Index &index, QueryBase::Comparator filter)
+static QVector<Identifier> sortedIndexLookup(Index &index, QueryBase::Comparator filter)
 {
     if (filter.comparator == Query::Comparator::In || filter.comparator == Query::Comparator::Contains) {
         SinkWarning() << "In and Contains comparison not supported on sorted indexes";
@@ -303,7 +303,7 @@ static QVector<QByteArray> sortedIndexLookup(Index &index, QueryBase::Comparator
         return indexLookup(index, filter, toSortableByteArray);
     }
 
-    QVector<QByteArray> keys;
+    QVector<Identifier> keys;
 
     QByteArray lowerBound, upperBound;
     auto bounds = filter.value.value<QVariantList>();
@@ -318,7 +318,7 @@ static QVector<QByteArray> sortedIndexLookup(Index &index, QueryBase::Comparator
 
     index.rangeLookup(lowerBound, upperBound,
         [&](const QByteArray &value) {
-            keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
+            keys << Identifier::fromInternalByteArray(value);
         },
         [bounds](const Index::Error &error) {
             SinkWarning() << "Lookup error in index:" << error.message
@@ -328,14 +328,14 @@ static QVector<QByteArray> sortedIndexLookup(Index &index, QueryBase::Comparator
     return keys;
 }
 
-static QVector<QByteArray> sampledIndexLookup(Index &index, QueryBase::Comparator filter)
+static QVector<Identifier> sampledIndexLookup(Index &index, QueryBase::Comparator filter)
 {
     if (filter.comparator != Query::Comparator::Overlap) {
         SinkWarning() << "Comparisons other than Overlap not supported on sampled period indexes";
         return {};
     }
 
-    QVector<QByteArray> keys;
+    QVector<Identifier> keys;
 
     auto bounds = filter.value.value<QVariantList>();
 
@@ -349,7 +349,7 @@ static QVector<QByteArray> sampledIndexLookup(Index &index, QueryBase::Comparato
 
     index.rangeLookup(lowerBucket, upperBucket,
         [&](const QByteArray &value) {
-            keys << Identifier::fromInternalByteArray(value).toDisplayByteArray();
+            keys << Identifier::fromInternalByteArray(value);
         },
         [bounds](const Index::Error &error) {
             SinkWarning() << "Lookup error in index:" << error.message
@@ -359,13 +359,18 @@ static QVector<QByteArray> sampledIndexLookup(Index &index, QueryBase::Comparato
     return keys;
 }
 
-QVector<QByteArray> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
+QVector<Identifier> TypeIndex::query(const Sink::QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId)
 {
     const auto baseFilters = query.getBaseFilters();
     for (auto it = baseFilters.constBegin(); it != baseFilters.constEnd(); it++) {
         if (it.value().comparator == QueryBase::Comparator::Fulltext) {
             FulltextIndex fulltextIndex{resourceInstanceId};
-            const auto keys = fulltextIndex.lookup(it.value().value.toString());
+            QVector<Identifier> keys;
+            const auto ids = fulltextIndex.lookup(it.value().value.toString());
+            keys.reserve(ids.size());
+            for (const auto &id : ids) {
+                keys.append(Identifier::fromDisplayByteArray(id));
+            }
             appliedFilters << it.key();
             SinkTraceCtx(mLogCtx) << "Fulltext index lookup found " << keys.size() << " keys.";
             return keys;
diff --git a/common/typeindex.h b/common/typeindex.h
index f2cabaf..875eb7a 100644
--- a/common/typeindex.h
+++ b/common/typeindex.h
@@ -94,7 +94,7 @@ public:
     void modify(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &oldEntity, const Sink::ApplicationDomain::ApplicationDomainType &newEntity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
     void remove(const Sink::Storage::Identifier &identifier, const Sink::ApplicationDomain::ApplicationDomainType &entity, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
 
-    QVector<QByteArray> query(const Sink::QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
+    QVector<Sink::Storage::Identifier> query(const Sink::QueryBase &query, QSet<QByteArrayList> &appliedFilters, QByteArray &appliedSorting, Sink::Storage::DataStore::Transaction &transaction, const QByteArray &resourceInstanceId);
     QVector<Sink::Storage::Identifier> lookup(const QByteArray &property, const QVariant &value, Sink::Storage::DataStore::Transaction &transaction);
 
     template <typename Left, typename Right>