Separate UIDs and Revisions in main databases

Summary:
- Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine)
- Better support for database flags (duplicate, integer keys, integer values for now but is extensible)
- Main databases' keys are now revisions
- Some databases switched to integer keys databases:
  - Main databases
  - the revision to uid mapping database
  - the revision to entity type mapping database
- Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again)
- New uid to revision mapping database (`uidsToRevisions`):
  - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions
  - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid)

~~Problems to fix before merging:~~
All Fixed!

- ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~
  - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~
- ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~

~~The first problem prevents me from running benchmarks~~

Reviewers: cmollekopf

Tags: #sink

Differential Revision: https://phabricator.kde.org/D14974

13 files changed, 344 insertions, 180 deletions

diff --git a/common/changereplay.cpp b/common/changereplay.cpp
index d7f46dc..96162b8 100644
--- a/common/changereplay.cpp
+++ b/common/changereplay.cpp
@@ -116,16 +116,15 @@ KAsync::Job<void> ChangeReplay::replayNextRevision()
                         } else {
                             // TODO: should not use internal representations
                             const auto key = Storage::Key(Storage::Identifier::fromDisplayByteArray(uid), revision);
-                            const auto internalKey = key.toInternalByteArray();
                             const auto displayKey = key.toDisplayByteArray();
                             QByteArray entityBuffer;
                             DataStore::mainDatabase(mMainStoreTransaction, type)
-                                .scan(internalKey,
-                                    [&entityBuffer](const QByteArray &key, const QByteArray &value) -> bool {
+                                .scan(revision,
+                                    [&entityBuffer](const size_t, const QByteArray &value) -> bool {
                                         entityBuffer = value;
                                         return false;
                                     },
-                                    [this, key](const DataStore::Error &) { SinkErrorCtx(mLogCtx) << "Failed to read the entity buffer " << key; });
+                                    [this, key](const DataStore::Error &e) { SinkErrorCtx(mLogCtx) << "Failed to read the entity buffer " << key << "error:" << e; });
 
                             if (entityBuffer.isEmpty()) {
                                 SinkErrorCtx(mLogCtx) << "Failed to replay change " << key;
diff --git a/common/domain/typeimplementations.cpp b/common/domain/typeimplementations.cpp
index aedf889..6e14501 100644
--- a/common/domain/typeimplementations.cpp
+++ b/common/domain/typeimplementations.cpp
@@ -80,7 +80,11 @@ typedef IndexConfig<Calendar,
         ValueIndex<Calendar::Name>
     > CalendarIndexConfig;
 
-
+template <typename EntityType, typename EntityIndexConfig>
+QMap<QByteArray, int> defaultTypeDatabases()
+{
+    return merge(QMap<QByteArray, int>{{QByteArray{EntityType::name} + ".main", Storage::IntegerKeys}}, EntityIndexConfig::databases());
+}
 
 void TypeImplementation<Mail>::configure(TypeIndex &index)
 {
@@ -89,7 +93,7 @@ void TypeImplementation<Mail>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Mail>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Mail::name} + ".main", 0}}, MailIndexConfig::databases());
+    return defaultTypeDatabases<Mail, MailIndexConfig>();
 }
 
 void TypeImplementation<Mail>::configure(IndexPropertyMapper &indexPropertyMapper)
@@ -132,7 +136,7 @@ void TypeImplementation<Folder>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Folder>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Folder::name} + ".main", 0}}, FolderIndexConfig::databases());
+    return defaultTypeDatabases<Folder, FolderIndexConfig>();
 }
 
 void TypeImplementation<Folder>::configure(PropertyMapper &propertyMapper)
@@ -157,7 +161,7 @@ void TypeImplementation<Contact>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Contact>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Contact::name} + ".main", 0}}, ContactIndexConfig::databases());
+    return defaultTypeDatabases<Contact, ContactIndexConfig>();
 }
 
 void TypeImplementation<Contact>::configure(PropertyMapper &propertyMapper)
@@ -185,7 +189,7 @@ void TypeImplementation<Addressbook>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Addressbook>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Addressbook::name} + ".main", 0}}, AddressbookIndexConfig::databases());
+    return defaultTypeDatabases<Addressbook, AddressbookIndexConfig>();
 }
 
 void TypeImplementation<Addressbook>::configure(PropertyMapper &propertyMapper)
@@ -207,7 +211,7 @@ void TypeImplementation<Event>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Event>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Event::name} + ".main", 0}}, EventIndexConfig::databases());
+    return defaultTypeDatabases<Event, EventIndexConfig>();
 }
 
 void TypeImplementation<Event>::configure(PropertyMapper &propertyMapper)
@@ -235,7 +239,7 @@ void TypeImplementation<Todo>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Todo>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Todo::name} + ".main", 0}}, TodoIndexConfig::databases());
+    return defaultTypeDatabases<Todo, TodoIndexConfig>();
 }
 
 void TypeImplementation<Todo>::configure(PropertyMapper &propertyMapper)
@@ -266,7 +270,7 @@ void TypeImplementation<Calendar>::configure(TypeIndex &index)
 
 QMap<QByteArray, int> TypeImplementation<Calendar>::typeDatabases()
 {
-    return merge(QMap<QByteArray, int>{{QByteArray{Calendar::name} + ".main", 0}}, CalendarIndexConfig::databases());
+    return defaultTypeDatabases<Calendar, CalendarIndexConfig>();
 }
 
 void TypeImplementation<Calendar>::configure(PropertyMapper &propertyMapper)
diff --git a/common/domain/typeimplementations_p.h b/common/domain/typeimplementations_p.h
index 51af113..bfdea77 100644
--- a/common/domain/typeimplementations_p.h
+++ b/common/domain/typeimplementations_p.h
@@ -57,7 +57,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + Property::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + Property::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -74,7 +74,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + Property::name + ".sort." + SortProperty::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -90,7 +90,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", 1}};
+        return {{QByteArray{EntityType::name} +".index." + SortProperty::name + ".sorted", Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -106,7 +106,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + Property::name + SecondaryProperty::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
@@ -142,7 +142,7 @@ public:
     template <typename EntityType>
     static QMap<QByteArray, int> databases()
     {
-        return {{QByteArray{EntityType::name} +".index." + RangeBeginProperty::name + ".range." + RangeEndProperty::name, 1}};
+        return {{QByteArray{EntityType::name} +".index." + RangeBeginProperty::name + ".range." + RangeEndProperty::name, Sink::Storage::AllowDuplicates}};
     }
 };
 
diff --git a/common/index.cpp b/common/index.cpp
index 238a745..bf8fcfc 100644
--- a/common/index.cpp
+++ b/common/index.cpp
@@ -6,7 +6,7 @@ using Sink::Storage::Identifier;
 
 Index::Index(const QString &storageRoot, const QString &dbName, const QString &indexName, Sink::Storage::DataStore::AccessMode mode)
     : mTransaction(Sink::Storage::DataStore(storageRoot, dbName, mode).createTransaction(mode)),
-      mDb(mTransaction.openDatabase(indexName.toLatin1(), std::function<void(const Sink::Storage::DataStore::Error &)>(), true)),
+      mDb(mTransaction.openDatabase(indexName.toLatin1(), std::function<void(const Sink::Storage::DataStore::Error &)>(), Sink::Storage::AllowDuplicates)),
       mName(indexName),
       mLogCtx("index." + indexName.toLatin1())
 {
@@ -14,7 +14,7 @@ Index::Index(const QString &storageRoot, const QString &dbName, const QString &i
 
 Index::Index(const QString &storageRoot, const QString &name, Sink::Storage::DataStore::AccessMode mode)
     : mTransaction(Sink::Storage::DataStore(storageRoot, name, mode).createTransaction(mode)),
-      mDb(mTransaction.openDatabase(name.toLatin1(), std::function<void(const Sink::Storage::DataStore::Error &)>(), true)),
+      mDb(mTransaction.openDatabase(name.toLatin1(), std::function<void(const Sink::Storage::DataStore::Error &)>(), Sink::Storage::AllowDuplicates)),
       mName(name),
       mLogCtx("index." + name.toLatin1())
 {
@@ -22,14 +22,14 @@ Index::Index(const QString &storageRoot, const QString &name, Sink::Storage::Dat
 
 Index::Index(const QString &storageRoot, const Sink::Storage::DbLayout &layout, Sink::Storage::DataStore::AccessMode mode)
     : mTransaction(Sink::Storage::DataStore(storageRoot, layout, mode).createTransaction(mode)),
-      mDb(mTransaction.openDatabase(layout.name, std::function<void(const Sink::Storage::DataStore::Error &)>(), true)),
+      mDb(mTransaction.openDatabase(layout.name, std::function<void(const Sink::Storage::DataStore::Error &)>(), Sink::Storage::AllowDuplicates)),
       mName(layout.name),
       mLogCtx("index." + layout.name)
 {
 }
 
 Index::Index(const QByteArray &name, Sink::Storage::DataStore::Transaction &transaction)
-    : mDb(transaction.openDatabase(name, std::function<void(const Sink::Storage::DataStore::Error &)>(), true)), mName(name),
+    : mDb(transaction.openDatabase(name, std::function<void(const Sink::Storage::DataStore::Error &)>(), Sink::Storage::AllowDuplicates)), mName(name),
       mLogCtx("index." + name)
 {
 }
diff --git a/common/mail/threadindexer.cpp b/common/mail/threadindexer.cpp
index c1d1aa8..b9de266 100644
--- a/common/mail/threadindexer.cpp
+++ b/common/mail/threadindexer.cpp
@@ -118,7 +118,7 @@ void ThreadIndexer::remove(const ApplicationDomain::ApplicationDomainType &entit
 
 QMap<QByteArray, int> ThreadIndexer::databases()
 {
-    return {{"mail.index.messageIdthreadId", 1},
-            {"mail.index.threadIdmessageId", 1}};
+    return {{"mail.index.messageIdthreadId", Sink::Storage::AllowDuplicates},
+            {"mail.index.threadIdmessageId", Sink::Storage::AllowDuplicates}};
 }
 
diff --git a/common/storage.h b/common/storage.h
index 8904148..ac6509d 100644
--- a/common/storage.h
+++ b/common/storage.h
@@ -32,6 +32,11 @@
 namespace Sink {
 namespace Storage {
 
+extern int AllowDuplicates;
+extern int IntegerKeys;
+// Only useful with AllowDuplicates
+extern int IntegerValues;
+
 struct SINK_EXPORT DbLayout {
     typedef QMap<QByteArray, int> Databases;
     DbLayout();
@@ -80,15 +85,23 @@ public:
          */
         bool write(const QByteArray &key, const QByteArray &value, const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>());
 
+        // of QByteArray for keys
+        bool write(const size_t key, const QByteArray &value, const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>());
+
         /**
          * Remove a key
          */
         void remove(const QByteArray &key, const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>());
+
+        void remove(const size_t key, const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>());
+
         /**
          * Remove a key-value pair
          */
         void remove(const QByteArray &key, const QByteArray &value, const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>());
 
+        void remove(const size_t key, const QByteArray &value, const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>());
+
         /**
         * Read values with a given key.
         *
@@ -101,6 +114,9 @@ public:
         int scan(const QByteArray &key, const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler,
             const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>(), bool findSubstringKeys = false, bool skipInternalKeys = true) const;
 
+        int scan(const size_t key, const std::function<bool(size_t key, const QByteArray &value)> &resultHandler,
+            const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>(), bool skipInternalKeys = true) const;
+
         /**
          * Finds the last value in a series matched by prefix.
          *
@@ -110,6 +126,9 @@ public:
         void findLatest(const QByteArray &uid, const std::function<void(const QByteArray &key, const QByteArray &value)> &resultHandler,
             const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>()) const;
 
+        void findLatest(size_t key, const std::function<void(size_t key, const QByteArray &value)> &resultHandler,
+            const std::function<void(const DataStore::Error &error)> &errorHandler = std::function<void(const DataStore::Error &error)>()) const;
+
         /**
          * Finds all the keys and values whose keys are in a given range
          * (inclusive).
@@ -119,6 +138,10 @@ public:
             const std::function<void(const DataStore::Error &error)> &errorHandler =
                 std::function<void(const DataStore::Error &error)>()) const;
 
+        int findAllInRange(const size_t lowerBound, const size_t upperBound,
+            const std::function<void(size_t key, const QByteArray &value)> &resultHandler,
+            const std::function<void(const DataStore::Error &error)> &errorHandler = {}) const;
+
         /**
          * Returns true if the database contains the substring key.
          */
@@ -163,8 +186,9 @@ public:
 
         QList<QByteArray> getDatabaseNames() const;
 
-        NamedDatabase openDatabase(const QByteArray &name = {"default"},
-            const std::function<void(const DataStore::Error &error)> &errorHandler = {}, bool allowDuplicates = false) const;
+        NamedDatabase openDatabase(const QByteArray &name = { "default" },
+            const std::function<void(const DataStore::Error &error)> &errorHandler = {},
+            int flags = 0) const;
 
         Transaction(Transaction &&other);
         Transaction &operator=(Transaction &&other);
@@ -224,13 +248,17 @@ public:
     static qint64 cleanedUpRevision(const Transaction &);
     static void setCleanedUpRevision(Transaction &, qint64 revision);
 
-    static QByteArray getUidFromRevision(const Transaction &, qint64 revision);
-    static QByteArray getTypeFromRevision(const Transaction &, qint64 revision);
-    static void recordRevision(Transaction &, qint64 revision, const QByteArray &uid, const QByteArray &type);
-    static void removeRevision(Transaction &, qint64 revision);
+    static QByteArray getUidFromRevision(const Transaction &, size_t revision);
+    static size_t getLatestRevisionFromUid(Transaction &, const QByteArray &uid);
+    static QList<size_t> getRevisionsUntilFromUid(DataStore::Transaction &, const QByteArray &uid, size_t lastRevision);
+    static QList<size_t> getRevisionsFromUid(DataStore::Transaction &, const QByteArray &uid);
+    static QByteArray getTypeFromRevision(const Transaction &, size_t revision);
+    static void recordRevision(Transaction &, size_t revision, const QByteArray &uid, const QByteArray &type);
+    static void removeRevision(Transaction &, size_t revision);
     static void recordUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type);
     static void removeUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type);
     static void getUids(const QByteArray &type, const Transaction &, const std::function<void(const QByteArray &uid)> &);
+    static bool hasUid(const QByteArray &type, const Transaction &, const QByteArray &uid);
 
     bool exists() const;
     static bool exists(const QString &storageRoot, const QString &name);
diff --git a/common/storage/entitystore.cpp b/common/storage/entitystore.cpp
index 276ee6a..454e25a 100644
--- a/common/storage/entitystore.cpp
+++ b/common/storage/entitystore.cpp
@@ -38,8 +38,9 @@ using namespace Sink::Storage;
 
 static QMap<QByteArray, int> baseDbs()
 {
-    return {{"revisionType", 0},
-            {"revisions", 0},
+    return {{"revisionType", Storage::IntegerKeys},
+            {"revisions", Storage::IntegerKeys},
+            {"uidsToRevisions", Storage::AllowDuplicates | Storage::IntegerValues},
             {"uids", 0},
             {"default", 0},
             {"__flagtable", 0}};
@@ -242,12 +243,13 @@ bool EntityStore::add(const QByteArray &type, ApplicationDomainType entity, bool
     const auto key = Key(identifier, newRevision);
 
     DataStore::mainDatabase(d->transaction, type)
-        .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb),
+        .write(newRevision, BufferUtils::extractBuffer(fbb),
             [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << entity.identifier() << newRevision; });
+
     DataStore::setMaxRevision(d->transaction, newRevision);
     DataStore::recordRevision(d->transaction, newRevision, entity.identifier(), type);
     DataStore::recordUid(d->transaction, entity.identifier(), type);
-    SinkTraceCtx(d->logCtx) << "Wrote entity: " << entity.identifier() << type << newRevision;
+    SinkTraceCtx(d->logCtx) << "Wrote entity: " << key << "of type:" << type;
     return true;
 }
 
@@ -319,8 +321,9 @@ bool EntityStore::modify(const QByteArray &type, const ApplicationDomainType &cu
     const auto key = Key(identifier, newRevision);
 
     DataStore::mainDatabase(d->transaction, type)
-        .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb),
+        .write(newRevision, BufferUtils::extractBuffer(fbb),
             [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << newEntity.identifier() << newRevision; });
+
     DataStore::setMaxRevision(d->transaction, newRevision);
     DataStore::recordRevision(d->transaction, newRevision, newEntity.identifier(), type);
     SinkTraceCtx(d->logCtx) << "Wrote modified entity: " << newEntity.identifier() << type << newRevision;
@@ -356,8 +359,9 @@ bool EntityStore::remove(const QByteArray &type, const ApplicationDomainType &cu
     const auto key = Key(identifier, newRevision);
 
     DataStore::mainDatabase(d->transaction, type)
-        .write(key.toInternalByteArray(), BufferUtils::extractBuffer(fbb),
+        .write(newRevision, BufferUtils::extractBuffer(fbb),
             [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to write entity" << uid << newRevision; });
+
     DataStore::setMaxRevision(d->transaction, newRevision);
     DataStore::recordRevision(d->transaction, newRevision, uid, type);
     DataStore::removeUid(d->transaction, uid, type);
@@ -375,30 +379,33 @@ void EntityStore::cleanupEntityRevisionsUntil(qint64 revision)
     }
     SinkTraceCtx(d->logCtx) << "Cleaning up revision " << revision << uid << bufferType;
     const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray();
-    DataStore::mainDatabase(d->transaction, bufferType)
-        .scan(internalUid,
-            [&](const QByteArray &key, const QByteArray &data) -> bool {
-                EntityBuffer buffer(const_cast<const char *>(data.data()), data.size());
-                if (!buffer.isValid()) {
-                    SinkWarningCtx(d->logCtx) << "Read invalid buffer from disk";
-                } else {
-                    const auto metadata = flatbuffers::GetRoot<Metadata>(buffer.metadataBuffer());
-                    const qint64 rev = metadata->revision();
-                    const auto isRemoval = metadata->operation() == Operation_Removal;
-                    // Remove old revisions, and the current if the entity has already been removed
-                    if (rev < revision || isRemoval) {
-                        DataStore::removeRevision(d->transaction, rev);
-                        DataStore::mainDatabase(d->transaction, bufferType).remove(key);
-                    }
-                    //Don't cleanup more than specified
-                    if (rev >= revision) {
-                        return false;
-                    }
-                }
 
-                return true;
-            },
-            [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error while reading: " << error.message; }, true);
+    // Remove old revisions
+    const auto revisionsToRemove = DataStore::getRevisionsUntilFromUid(d->transaction, uid, revision);
+
+    for (const auto &revisionToRemove : revisionsToRemove) {
+        DataStore::removeRevision(d->transaction, revisionToRemove);
+        DataStore::mainDatabase(d->transaction, bufferType).remove(revisionToRemove);
+    }
+
+    // And remove the specified revision only if marked for removal
+    DataStore::mainDatabase(d->transaction, bufferType).scan(revision, [&](size_t, const QByteArray &data) {
+        EntityBuffer buffer(const_cast<const char *>(data.data()), data.size());
+        if (!buffer.isValid()) {
+            SinkWarningCtx(d->logCtx) << "Read invalid buffer from disk";
+            return false;
+        }
+
+        const auto metadata = flatbuffers::GetRoot<Metadata>(buffer.metadataBuffer());
+        const qint64 rev = metadata->revision();
+        if (metadata->operation() == Operation_Removal) {
+            DataStore::removeRevision(d->transaction, revision);
+            DataStore::mainDatabase(d->transaction, bufferType).remove(revision);
+        }
+
+        return false;
+    });
+
     DataStore::setCleanedUpRevision(d->transaction, revision);
 }
 
@@ -433,20 +440,12 @@ QVector<Identifier> EntityStore::fullScan(const QByteArray &type)
         SinkTraceCtx(d->logCtx) << "Database is not existing: " << type;
         return {};
     }
-    //The scan can return duplicate results if we have multiple revisions, so we use a set to deduplicate.
+
     QSet<Identifier> keys;
-    DataStore::mainDatabase(d->getTransaction(), type)
-        .scan(QByteArray(),
-            [&](const QByteArray &key, const QByteArray &value) -> bool {
-                const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier();
-                if (keys.contains(uid)) {
-                    //Not something that should persist if the replay works, so we keep a message for now.
-                    SinkTraceCtx(d->logCtx) << "Multiple revisions for uid: " << Sink::Storage::Key::fromInternalByteArray(key) << ". This is normal if changereplay has not completed yet.";
-                }
-                keys << uid;
-                return true;
-            },
-            [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during fullScan query: " << error.message; });
+
+    DataStore::getUids(type, d->getTransaction(), [&keys] (const QByteArray &uid) {
+        keys << Identifier::fromDisplayByteArray(uid);
+    });
 
     SinkTraceCtx(d->logCtx) << "Full scan retrieved " << keys.size() << " results.";
     return keys.toList().toVector();
@@ -492,12 +491,12 @@ void EntityStore::indexLookup(const QByteArray &type, const QByteArray &property
 void EntityStore::readLatest(const QByteArray &type, const Identifier &id, const std::function<void(const QByteArray &uid, const EntityBuffer &entity)> callback)
 {
     Q_ASSERT(d);
-    const auto internalKey = id.toInternalByteArray();
+    const size_t revision = DataStore::getLatestRevisionFromUid(d->getTransaction(), id.toDisplayByteArray());
     auto db = DataStore::mainDatabase(d->getTransaction(), type);
-    db.findLatest(internalKey,
-        [=](const QByteArray &key, const QByteArray &value) {
-            const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray();
-            callback(uid, Sink::EntityBuffer(value.data(), value.size()));
+    db.scan(revision,
+        [=](size_t, const QByteArray &value) {
+            callback(id.toDisplayByteArray(), Sink::EntityBuffer(value.data(), value.size()));
+            return false;
         },
         [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error during readLatest query: " << error.message << id; });
 }
@@ -546,9 +545,9 @@ void EntityStore::readEntity(const QByteArray &type, const QByteArray &displayKe
 {
     const auto key = Key::fromDisplayByteArray(displayKey);
     auto db = DataStore::mainDatabase(d->getTransaction(), type);
-    db.scan(key.toInternalByteArray(),
-        [=](const QByteArray &key, const QByteArray &value) -> bool {
-            const auto uid = Sink::Storage::Key::fromInternalByteArray(key).identifier().toDisplayByteArray();
+    db.scan(key.revision().toSizeT(),
+        [=](size_t rev, const QByteArray &value) -> bool {
+            const auto uid = DataStore::getUidFromRevision(d->transaction, rev);
             callback(uid, Sink::EntityBuffer(value.data(), value.size()));
             return false;
         },
@@ -604,18 +603,8 @@ void EntityStore::readRevisions(qint64 baseRevision, const QByteArray &expectedT
 
 void EntityStore::readPrevious(const QByteArray &type, const Identifier &id, qint64 revision, const std::function<void(const QByteArray &uid, const EntityBuffer &entity)> callback)
 {
-    auto db = DataStore::mainDatabase(d->getTransaction(), type);
-    qint64 latestRevision = 0;
-    const auto internalUid = id.toInternalByteArray();
-    db.scan(internalUid,
-        [&latestRevision, revision](const QByteArray &key, const QByteArray &) -> bool {
-            const auto foundRevision = Key::fromInternalByteArray(key).revision().toQint64();
-            if (foundRevision < revision && foundRevision > latestRevision) {
-                latestRevision = foundRevision;
-            }
-            return true;
-        },
-        [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read current value from storage: " << error.message; }, true);
+    const auto previousRevisions = DataStore::getRevisionsUntilFromUid(d->getTransaction(), id.toDisplayByteArray(), revision);
+    const size_t latestRevision = previousRevisions[previousRevisions.size() - 1];
     const auto key = Key(id, latestRevision);
     readEntity(type, key.toDisplayByteArray(), callback);
 }
@@ -641,21 +630,20 @@ void EntityStore::readAllUids(const QByteArray &type, const std::function<void(c
     DataStore::getUids(type, d->getTransaction(), callback);
 }
 
-bool EntityStore::contains(const QByteArray &type, const QByteArray &uid)
+bool EntityStore::contains(const QByteArray & /* type */, const QByteArray &uid)
 {
     Q_ASSERT(!uid.isEmpty());
-    const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray();
-    return DataStore::mainDatabase(d->getTransaction(), type).contains(internalUid);
+    return !DataStore::getRevisionsFromUid(d->getTransaction(), uid).isEmpty();
 }
 
 bool EntityStore::exists(const QByteArray &type, const QByteArray &uid)
 {
     bool found = false;
     bool alreadyRemoved = false;
-    const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray();
+    const size_t revision = DataStore::getLatestRevisionFromUid(d->getTransaction(), uid);
     DataStore::mainDatabase(d->transaction, type)
-        .findLatest(internalUid,
-            [&found, &alreadyRemoved](const QByteArray &key, const QByteArray &data) {
+        .scan(revision,
+            [&found, &alreadyRemoved](size_t, const QByteArray &data) {
                 auto entity = GetEntity(data.data());
                 if (entity && entity->metadata()) {
                     auto metadata = GetMetadata(entity->metadata()->Data());
@@ -664,6 +652,7 @@ bool EntityStore::exists(const QByteArray &type, const QByteArray &uid)
                         alreadyRemoved = true;
                     }
                 }
+                return true;
             },
             [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Failed to read old revision from storage: " << error.message; });
     if (!found) {
@@ -677,23 +666,32 @@ bool EntityStore::exists(const QByteArray &type, const QByteArray &uid)
     return true;
 }
 
-void EntityStore::readRevisions(const QByteArray &type, const QByteArray &uid, qint64 startingRevision, const std::function<void(const QByteArray &uid, qint64 revision, const EntityBuffer &entity)> callback)
+void EntityStore::readRevisions(const QByteArray &type, const QByteArray &uid, qint64 startingRevision,
+    const std::function<void(const QByteArray &uid, qint64 revision, const EntityBuffer &entity)> callback)
 {
     Q_ASSERT(d);
     Q_ASSERT(!uid.isEmpty());
-    const auto internalUid = Identifier::fromDisplayByteArray(uid).toInternalByteArray();
-    DataStore::mainDatabase(d->transaction, type)
-        .scan(internalUid,
-            [&](const QByteArray &key, const QByteArray &value) -> bool {
-                const auto parsedKey = Key::fromInternalByteArray(key);
-                const auto revision = parsedKey.revision().toQint64();
-                if (revision >= startingRevision) {
-                    callback(parsedKey.identifier().toDisplayByteArray(), revision, Sink::EntityBuffer(value.data(), value.size()));
-                }
-                return true;
-            },
-            [&](const DataStore::Error &error) { SinkWarningCtx(d->logCtx) << "Error while reading: " << error.message; }, true);
 
+    const auto revisions = DataStore::getRevisionsFromUid(d->transaction, uid);
+
+    const auto db = DataStore::mainDatabase(d->transaction, type);
+
+    for (const auto revision : revisions) {
+        if (revision < startingRevision) {
+            continue;
+        }
+
+        db.scan(revision,
+            [&](size_t rev, const QByteArray &value) {
+                Q_ASSERT(rev == revision);
+                callback(uid, revision, Sink::EntityBuffer(value.data(), value.size()));
+                return false;
+            },
+            [&](const DataStore::Error &error) {
+                SinkWarningCtx(d->logCtx) << "Error while reading: " << error.message;
+            },
+            true);
+    }
 }
 
 qint64 EntityStore::maxRevision()
diff --git a/common/storage/key.cpp b/common/storage/key.cpp
index 2327061..a6567ea 100644
--- a/common/storage/key.cpp
+++ b/common/storage/key.cpp
@@ -155,6 +155,11 @@ qint64 Revision::toQint64() const
     return rev;
 }
 
+size_t Revision::toSizeT() const
+{
+    return rev;
+}
+
 bool Revision::isValidInternal(const QByteArray &bytes)
 {
     if (bytes.size() != Revision::INTERNAL_REPR_SIZE) {
diff --git a/common/storage/key.h b/common/storage/key.h
index baabe38..da90ddd 100644
--- a/common/storage/key.h
+++ b/common/storage/key.h
@@ -67,7 +67,7 @@ public:
     static const constexpr size_t INTERNAL_REPR_SIZE = 19;
     static const constexpr size_t DISPLAY_REPR_SIZE = 19;
 
-    Revision(qint64 rev) : rev(rev) {}
+    Revision(size_t rev) : rev(rev) {}
 
     QByteArray toInternalByteArray() const;
     static Revision fromInternalByteArray(const QByteArray &bytes);
@@ -75,6 +75,7 @@ public:
     QByteArray toDisplayByteArray() const;
     static Revision fromDisplayByteArray(const QByteArray &bytes);
     qint64 toQint64() const;
+    size_t toSizeT() const;
 
     static bool isValidInternal(const QByteArray &);
     static bool isValidDisplay(const QByteArray &);
@@ -84,7 +85,7 @@ public:
     bool operator!=(const Revision &other) const;
 
 private:
-    qint64 rev;
+    size_t rev;
 };
 
 class Key
diff --git a/common/storage_common.cpp b/common/storage_common.cpp
index 264f223..7c794c3 100644
--- a/common/storage_common.cpp
+++ b/common/storage_common.cpp
@@ -117,26 +117,69 @@ qint64 DataStore::cleanedUpRevision(const DataStore::Transaction &transaction)
     return r;
 }
 
-QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transaction, qint64 revision)
+QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transaction, size_t revision)
 {
     QByteArray uid;
-    transaction.openDatabase("revisions")
-        .scan(QByteArray::number(revision),
-            [&](const QByteArray &, const QByteArray &value) -> bool {
-                uid = QByteArray{value.constData(), value.size()};
+    transaction
+        .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys)
+        .scan(revision,
+            [&](const size_t, const QByteArray &value) -> bool {
+                uid = QByteArray{ value.constData(), value.size() };
                 return false;
             },
-            [revision](const Error &error) { SinkWarning() << "Couldn't find uid for revision: " << revision << error.message; });
+            [revision](const Error &error) {
+                SinkWarning() << "Couldn't find uid for revision: " << revision << error.message;
+            });
     Q_ASSERT(!uid.isEmpty());
     return uid;
 }
 
-QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transaction, qint64 revision)
+size_t DataStore::getLatestRevisionFromUid(DataStore::Transaction &t, const QByteArray &uid)
+{
+    size_t revision;
+    t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues)
+        .findLatest(uid, [&revision](const QByteArray &key, const QByteArray &value) {
+            revision = byteArrayToSizeT(value);
+        });
+
+    return revision;
+}
+
+QList<size_t> DataStore::getRevisionsUntilFromUid(DataStore::Transaction &t, const QByteArray &uid, size_t lastRevision)
+{
+    QList<size_t> queriedRevisions;
+    t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues)
+        .scan(uid, [&queriedRevisions, lastRevision](const QByteArray &, const QByteArray &value) {
+            size_t currentRevision = byteArrayToSizeT(value);
+            if (currentRevision < lastRevision) {
+                queriedRevisions << currentRevision;
+                return true;
+            }
+
+            return false;
+        });
+
+    return queriedRevisions;
+}
+
+QList<size_t> DataStore::getRevisionsFromUid(DataStore::Transaction &t, const QByteArray &uid)
+{
+    QList<size_t> queriedRevisions;
+    t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues)
+        .scan(uid, [&queriedRevisions](const QByteArray &, const QByteArray &value) {
+            queriedRevisions << byteArrayToSizeT(value);
+            return true;
+        });
+
+    return queriedRevisions;
+}
+
+QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transaction, size_t revision)
 {
     QByteArray type;
-    transaction.openDatabase("revisionType")
-        .scan(QByteArray::number(revision),
-            [&](const QByteArray &, const QByteArray &value) -> bool {
+    transaction.openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys)
+        .scan(revision,
+            [&](const size_t, const QByteArray &value) -> bool {
                 type = QByteArray{value.constData(), value.size()};
                 return false;
             },
@@ -145,17 +188,31 @@ QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transact
     return type;
 }
 
-void DataStore::recordRevision(DataStore::Transaction &transaction, qint64 revision, const QByteArray &uid, const QByteArray &type)
+void DataStore::recordRevision(DataStore::Transaction &transaction, size_t revision,
+    const QByteArray &uid, const QByteArray &type)
 {
-    // TODO use integerkeys
-    transaction.openDatabase("revisions").write(QByteArray::number(revision), uid);
-    transaction.openDatabase("revisionType").write(QByteArray::number(revision), type);
+    transaction
+        .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys)
+        .write(revision, uid);
+    transaction.openDatabase("uidsToRevisions", /* errorHandler = */ {}, AllowDuplicates | IntegerValues)
+        .write(uid, sizeTToByteArray(revision));
+    transaction
+        .openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys)
+        .write(revision, type);
 }
 
-void DataStore::removeRevision(DataStore::Transaction &transaction, qint64 revision)
+void DataStore::removeRevision(DataStore::Transaction &transaction, size_t revision)
 {
-    transaction.openDatabase("revisions").remove(QByteArray::number(revision));
-    transaction.openDatabase("revisionType").remove(QByteArray::number(revision));
+    const QByteArray uid = getUidFromRevision(transaction, revision);
+
+    transaction
+        .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys)
+        .remove(revision);
+    transaction.openDatabase("uidsToRevisions", /* errorHandler = */ {}, AllowDuplicates | IntegerValues)
+        .remove(uid, sizeTToByteArray(revision));
+    transaction
+        .openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys)
+        .remove(revision);
 }
 
 void DataStore::recordUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type)
@@ -176,6 +233,18 @@ void DataStore::getUids(const QByteArray &type, const Transaction &transaction,
     });
 }
 
+bool DataStore::hasUid(const QByteArray &type, const Transaction &transaction, const QByteArray &uid)
+{
+    bool hasTheUid = false;
+    transaction.openDatabase(type + "uids").scan(uid, [&](const QByteArray &key, const QByteArray &) {
+        Q_ASSERT(uid == key);
+        hasTheUid = true;
+        return false;
+    });
+
+    return hasTheUid;
+}
+
 bool DataStore::isInternalKey(const char *key)
 {
     return key && strncmp(key, s_internalPrefix, s_internalPrefixSize) == 0;
@@ -207,7 +276,7 @@ DataStore::NamedDatabase DataStore::mainDatabase(const DataStore::Transaction &t
         Q_ASSERT(false);
         return {};
     }
-    return t.openDatabase(type + ".main");
+    return t.openDatabase(type + ".main", /* errorHandler= */ {}, IntegerKeys);
 }
 
 bool DataStore::NamedDatabase::contains(const QByteArray &uid)
diff --git a/common/storage_lmdb.cpp b/common/storage_lmdb.cpp
index a007405..0458dae 100644
--- a/common/storage_lmdb.cpp
+++ b/common/storage_lmdb.cpp
@@ -48,6 +48,10 @@ static QMutex sCreateDbiLock;
 static QHash<QString, MDB_env *> sEnvironments;
 static QHash<QString, MDB_dbi> sDbis;
 
+int AllowDuplicates = MDB_DUPSORT;
+int IntegerKeys = MDB_INTEGERKEY;
+int IntegerValues = MDB_INTEGERDUP;
+
 int getErrorCode(int e)
 {
     switch (e) {
@@ -101,14 +105,8 @@ static QList<QByteArray> getDatabaseNames(MDB_txn *transaction)
  * and we always need to commit the transaction ASAP
  * We can only ever enter from one point per process.
  */
-static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, bool allowDuplicates, MDB_dbi &dbi)
+static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly, int flags, MDB_dbi &dbi)
 {
-
-    unsigned int flags = 0;
-    if (allowDuplicates) {
-        flags |= MDB_DUPSORT;
-    }
-
     MDB_dbi flagtableDbi;
     if (const int rc = mdb_dbi_open(transaction, "__flagtable", readOnly ? 0 : MDB_CREATE, &flagtableDbi)) {
         if (!readOnly) {
@@ -130,6 +128,10 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly,
         }
     }
 
+    if (flags & IntegerValues && !(flags & AllowDuplicates)) {
+        SinkWarning() << "Opening a database with integer values, but not duplicate keys";
+    }
+
     if (const int rc = mdb_dbi_open(transaction, db.constData(), flags, &dbi)) {
         //Create the db if it is not existing already
         if (rc == MDB_NOTFOUND && !readOnly) {
@@ -165,7 +167,7 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly,
             //Store the flags without the create option
             const auto ba = QByteArray::number(flags);
             value.mv_data = const_cast<void*>(static_cast<const void*>(ba.constData()));
-            value.mv_size = db.size();
+            value.mv_size = ba.size();
             if (const int rc = mdb_put(transaction, flagtableDbi, &key, &value, MDB_NOOVERWRITE)) {
                 //We expect this to fail if we're only creating the dbi but not the db
                 if (rc != MDB_KEYEXIST) {
@@ -175,7 +177,7 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly,
         } else {
             //It's not an error if we only want to read
             if (!readOnly) {
-                SinkWarning() << "Failed to open db " << QByteArray(mdb_strerror(rc));
+                SinkWarning() << "Failed to open db " << db << "error:" << QByteArray(mdb_strerror(rc));
                 return true;
             }
             return false;
@@ -187,8 +189,14 @@ static bool createDbi(MDB_txn *transaction, const QByteArray &db, bool readOnly,
 class DataStore::NamedDatabase::Private
 {
 public:
-    Private(const QByteArray &_db, bool _allowDuplicates, const std::function<void(const DataStore::Error &error)> &_defaultErrorHandler, const QString &_name, MDB_txn *_txn)
-        : db(_db), transaction(_txn), allowDuplicates(_allowDuplicates), defaultErrorHandler(_defaultErrorHandler), name(_name)
+    Private(const QByteArray &_db, int _flags,
+        const std::function<void(const DataStore::Error &error)> &_defaultErrorHandler,
+        const QString &_name, MDB_txn *_txn)
+        : db(_db),
+          transaction(_txn),
+          flags(_flags),
+          defaultErrorHandler(_defaultErrorHandler),
+          name(_name)
     {
     }
 
@@ -199,7 +207,7 @@ public:
     QByteArray db;
     MDB_txn *transaction;
     MDB_dbi dbi;
-    bool allowDuplicates;
+    int flags;
     std::function<void(const DataStore::Error &error)> defaultErrorHandler;
     QString name;
     bool createdNewDbi = false;
@@ -313,7 +321,7 @@ public:
         } else {
             dbiTransaction = transaction;
         }
-        if (createDbi(dbiTransaction, db, readOnly, allowDuplicates, dbi)) {
+        if (createDbi(dbiTransaction, db, readOnly, flags, dbi)) {
             if (readOnly) {
                 mdb_txn_commit(dbiTransaction);
                 Q_ASSERT(!sDbis.contains(dbiName));
@@ -371,6 +379,12 @@ DataStore::NamedDatabase::~NamedDatabase()
     delete d;
 }
 
+bool DataStore::NamedDatabase::write(const size_t key, const QByteArray &value,
+    const std::function<void(const DataStore::Error &error)> &errorHandler)
+{
+    return write(sizeTToByteArray(key), value, errorHandler);
+}
+
 bool DataStore::NamedDatabase::write(const QByteArray &sKey, const QByteArray &sValue, const std::function<void(const DataStore::Error &error)> &errorHandler)
 {
     if (!d || !d->transaction) {
@@ -407,11 +421,23 @@ bool DataStore::NamedDatabase::write(const QByteArray &sKey, const QByteArray &s
     return !rc;
 }
 
+void DataStore::NamedDatabase::remove(
+    const size_t key, const std::function<void(const DataStore::Error &error)> &errorHandler)
+{
+    return remove(sizeTToByteArray(key), errorHandler);
+}
+
 void DataStore::NamedDatabase::remove(const QByteArray &k, const std::function<void(const DataStore::Error &error)> &errorHandler)
 {
     remove(k, QByteArray(), errorHandler);
 }
 
+void DataStore::NamedDatabase::remove(const size_t key, const QByteArray &value,
+    const std::function<void(const DataStore::Error &error)> &errorHandler)
+{
+    return remove(sizeTToByteArray(key), value, errorHandler);
+}
+
 void DataStore::NamedDatabase::remove(const QByteArray &k, const QByteArray &value, const std::function<void(const DataStore::Error &error)> &errorHandler)
 {
     if (!d || !d->transaction) {
@@ -445,6 +471,17 @@ void DataStore::NamedDatabase::remove(const QByteArray &k, const QByteArray &val
     }
 }
 
+int DataStore::NamedDatabase::scan(const size_t key,
+    const std::function<bool(size_t key, const QByteArray &value)> &resultHandler,
+    const std::function<void(const DataStore::Error &error)> &errorHandler, bool skipInternalKeys) const
+{
+    return scan(sizeTToByteArray(key),
+        [&resultHandler](const QByteArray &key, const QByteArray &value) {
+            return resultHandler(byteArrayToSizeT(key), value);
+        },
+        errorHandler, /* findSubstringKeys = */ false, skipInternalKeys);
+}
+
 int DataStore::NamedDatabase::scan(const QByteArray &k, const std::function<bool(const QByteArray &key, const QByteArray &value)> &resultHandler,
     const std::function<void(const DataStore::Error &error)> &errorHandler, bool findSubstringKeys, bool skipInternalKeys) const
 {
@@ -471,8 +508,10 @@ int DataStore::NamedDatabase::scan(const QByteArray &k, const std::function<bool
 
     int numberOfRetrievedValues = 0;
 
-    if (k.isEmpty() || d->allowDuplicates || findSubstringKeys) {
-        MDB_cursor_op op = d->allowDuplicates ? MDB_SET : MDB_FIRST;
+    bool allowDuplicates = d->flags & AllowDuplicates;
+
+    if (k.isEmpty() || allowDuplicates || findSubstringKeys) {
+        MDB_cursor_op op = allowDuplicates ? MDB_SET : MDB_FIRST;
         if (findSubstringKeys) {
             op = MDB_SET_RANGE;
         }
@@ -490,7 +529,7 @@ int DataStore::NamedDatabase::scan(const QByteArray &k, const std::function<bool
                         key.mv_data = (void *)k.constData();
                         key.mv_size = k.size();
                     }
-                    MDB_cursor_op nextOp = (d->allowDuplicates && !findSubstringKeys) ? MDB_NEXT_DUP : MDB_NEXT;
+                    MDB_cursor_op nextOp = (allowDuplicates && !findSubstringKeys) ? MDB_NEXT_DUP : MDB_NEXT;
                     while ((rc = mdb_cursor_get(cursor, &key, &data, nextOp)) == 0) {
                         const auto current = QByteArray::fromRawData((char *)key.mv_data, key.mv_size);
                         // Every consequitive lookup simply iterates through the list
@@ -529,6 +568,18 @@ int DataStore::NamedDatabase::scan(const QByteArray &k, const std::function<bool
     return numberOfRetrievedValues;
 }
 
+
+void DataStore::NamedDatabase::findLatest(size_t key,
+    const std::function<void(size_t key, const QByteArray &value)> &resultHandler,
+    const std::function<void(const DataStore::Error &error)> &errorHandler) const
+{
+    return findLatest(sizeTToByteArray(key),
+        [&resultHandler](const QByteArray &key, const QByteArray &value) {
+            resultHandler(byteArrayToSizeT(value), value);
+        },
+        errorHandler);
+}
+
 void DataStore::NamedDatabase::findLatest(const QByteArray &k, const std::function<void(const QByteArray &key, const QByteArray &value)> &resultHandler,
     const std::function<void(const DataStore::Error &error)> &errorHandler) const
 {
@@ -602,6 +653,17 @@ void DataStore::NamedDatabase::findLatest(const QByteArray &k, const std::functi
     return;
 }
 
+int DataStore::NamedDatabase::findAllInRange(const size_t lowerBound, const size_t upperBound,
+    const std::function<void(size_t key, const QByteArray &value)> &resultHandler,
+    const std::function<void(const DataStore::Error &error)> &errorHandler) const
+{
+    return findAllInRange(sizeTToByteArray(lowerBound), sizeTToByteArray(upperBound),
+        [&resultHandler](const QByteArray &key, const QByteArray &value) {
+            resultHandler(byteArrayToSizeT(value), value);
+        },
+        errorHandler);
+}
+
 int DataStore::NamedDatabase::findAllInRange(const QByteArray &lowerBound, const QByteArray &upperBound,
     const std::function<void(const QByteArray &key, const QByteArray &value)> &resultHandler,
     const std::function<void(const DataStore::Error &error)> &errorHandler) const
@@ -839,30 +901,8 @@ void DataStore::Transaction::abort()
     d->transaction = nullptr;
 }
 
-//Ensure that we opened the correct database by comparing the expected identifier with the one
-//we write to the database on first open.
-static bool ensureCorrectDb(DataStore::NamedDatabase &database, const QByteArray &db, bool readOnly)
-{
-    bool openedTheWrongDatabase = false;
-    auto count = database.scan("__internal_dbname", [db, &openedTheWrongDatabase](const QByteArray &key, const QByteArray &value) ->bool {
-        if (value != db) {
-            SinkWarning() << "Opened the wrong database, got " << value << " instead of " << db;
-            openedTheWrongDatabase = true;
-        }
-        return false;
-    },
-    [&](const DataStore::Error &) {
-    }, false);
-    //This is the first time we open this database in a write transaction, write the db name
-    if (!count) {
-        if (!readOnly) {
-            database.write("__internal_dbname", db);
-        }
-    }
-    return !openedTheWrongDatabase;
-}
-
-DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &db, const std::function<void(const DataStore::Error &error)> &errorHandler, bool allowDuplicates) const
+DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &db,
+    const std::function<void(const DataStore::Error &error)> &errorHandler, int flags) const
 {
     if (!d) {
         SinkError() << "Tried to open database on invalid transaction: " << db;
@@ -871,7 +911,8 @@ DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &
     Q_ASSERT(d->transaction);
     // We don't now if anything changed
     d->implicitCommit = true;
-    auto p = new DataStore::NamedDatabase::Private(db, allowDuplicates, d->defaultErrorHandler, d->name, d->transaction);
+    auto p = new DataStore::NamedDatabase::Private(
+        db, flags, d->defaultErrorHandler, d->name, d->transaction);
     auto ret = p->openDatabase(d->requestedRead, errorHandler);
     if (!ret) {
         delete p;
@@ -883,11 +924,6 @@ DataStore::NamedDatabase DataStore::Transaction::openDatabase(const QByteArray &
     }
 
     auto database = DataStore::NamedDatabase(p);
-    if (!ensureCorrectDb(database, db, d->requestedRead)) {
-        SinkWarning() << "Failed to open the database correctly" << db;
-        Q_ASSERT(false);
-        return DataStore::NamedDatabase();
-    }
     return database;
 }
 
@@ -1049,11 +1085,11 @@ public:
 
                             //Create dbis from the given layout.
                             for (auto it = layout.tables.constBegin(); it != layout.tables.constEnd(); it++) {
-                                const bool allowDuplicates = it.value();
+                                const int flags = it.value();
                                 MDB_dbi dbi = 0;
                                 const auto db = it.key();
                                 const auto dbiName = name + db;
-                                if (createDbi(transaction, db, readOnly, allowDuplicates, dbi)) {
+                                if (createDbi(transaction, db, readOnly, flags, dbi)) {
                                     sDbis.insert(dbiName, dbi);
                                 }
                             }
@@ -1063,8 +1099,8 @@ public:
                                 MDB_dbi dbi = 0;
                                 const auto dbiName = name + db;
                                 //We're going to load the flags anyways.
-                                bool allowDuplicates = false;
-                                if (createDbi(transaction, db, readOnly, allowDuplicates, dbi)) {
+                                const int flags = 0;
+                                if (createDbi(transaction, db, readOnly, flags, dbi)) {
                                     sDbis.insert(dbiName, dbi);
                                 }
                             }
diff --git a/common/utils.cpp b/common/utils.cpp
index 3c54db4..f6c6798 100644
--- a/common/utils.cpp
+++ b/common/utils.cpp
@@ -23,3 +23,13 @@ QByteArray Sink::createUuid()
 {
     return QUuid::createUuid().toByteArray();
 }
+
+const QByteArray Sink::sizeTToByteArray(const size_t &value)
+{
+    return QByteArray::fromRawData(reinterpret_cast<const char *>(&value), sizeof(size_t));
+}
+
+size_t Sink::byteArrayToSizeT(const QByteArray &value)
+{
+    return *reinterpret_cast<const size_t *>(value.constData());
+}
diff --git a/common/utils.h b/common/utils.h
index 7066d79..8565f17 100644
--- a/common/utils.h
+++ b/common/utils.h
@@ -26,6 +26,20 @@ namespace Sink {
 
 QByteArray createUuid();
 
+// No copy is done on this functions. Therefore, the caller must not use the
+// returned QByteArray after the size_t has been destroyed.
+const QByteArray sizeTToByteArray(const size_t &);
+size_t byteArrayToSizeT(const QByteArray &);
+
+template <typename T>
+static QByteArray padNumber(T number);
+
+template <>
+QByteArray padNumber<size_t>(size_t number)
+{
+    return padNumber<qint64>(number);
+}
+
 template <typename T>
 static QByteArray padNumber(T number)
 {