Separate UIDs and Revisions in main databases

Summary:
- Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine)
- Better support for database flags (duplicate, integer keys, integer values for now but is extensible)
- Main databases' keys are now revisions
- Some databases switched to integer keys databases:
  - Main databases
  - the revision to uid mapping database
  - the revision to entity type mapping database
- Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again)
- New uid to revision mapping database (`uidsToRevisions`):
  - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions
  - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid)

~~Problems to fix before merging:~~
All Fixed!

- ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~
  - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~
- ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~

~~The first problem prevents me from running benchmarks~~

Reviewers: cmollekopf

Tags: #sink

Differential Revision: https://phabricator.kde.org/D14974

1 files changed, 87 insertions, 18 deletions

diff --git a/common/storage_common.cpp b/common/storage_common.cpp
index 264f223..7c794c3 100644
--- a/common/storage_common.cpp
+++ b/common/storage_common.cpp
@@ -117,26 +117,69 @@ qint64 DataStore::cleanedUpRevision(const DataStore::Transaction &transaction)
     return r;
 }
 
-QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transaction, qint64 revision)
+QByteArray DataStore::getUidFromRevision(const DataStore::Transaction &transaction, size_t revision)
 {
     QByteArray uid;
-    transaction.openDatabase("revisions")
-        .scan(QByteArray::number(revision),
-            [&](const QByteArray &, const QByteArray &value) -> bool {
-                uid = QByteArray{value.constData(), value.size()};
+    transaction
+        .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys)
+        .scan(revision,
+            [&](const size_t, const QByteArray &value) -> bool {
+                uid = QByteArray{ value.constData(), value.size() };
                 return false;
             },
-            [revision](const Error &error) { SinkWarning() << "Couldn't find uid for revision: " << revision << error.message; });
+            [revision](const Error &error) {
+                SinkWarning() << "Couldn't find uid for revision: " << revision << error.message;
+            });
     Q_ASSERT(!uid.isEmpty());
     return uid;
 }
 
-QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transaction, qint64 revision)
+size_t DataStore::getLatestRevisionFromUid(DataStore::Transaction &t, const QByteArray &uid)
+{
+    size_t revision;
+    t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues)
+        .findLatest(uid, [&revision](const QByteArray &key, const QByteArray &value) {
+            revision = byteArrayToSizeT(value);
+        });
+
+    return revision;
+}
+
+QList<size_t> DataStore::getRevisionsUntilFromUid(DataStore::Transaction &t, const QByteArray &uid, size_t lastRevision)
+{
+    QList<size_t> queriedRevisions;
+    t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues)
+        .scan(uid, [&queriedRevisions, lastRevision](const QByteArray &, const QByteArray &value) {
+            size_t currentRevision = byteArrayToSizeT(value);
+            if (currentRevision < lastRevision) {
+                queriedRevisions << currentRevision;
+                return true;
+            }
+
+            return false;
+        });
+
+    return queriedRevisions;
+}
+
+QList<size_t> DataStore::getRevisionsFromUid(DataStore::Transaction &t, const QByteArray &uid)
+{
+    QList<size_t> queriedRevisions;
+    t.openDatabase("uidsToRevisions", {}, AllowDuplicates | IntegerValues)
+        .scan(uid, [&queriedRevisions](const QByteArray &, const QByteArray &value) {
+            queriedRevisions << byteArrayToSizeT(value);
+            return true;
+        });
+
+    return queriedRevisions;
+}
+
+QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transaction, size_t revision)
 {
     QByteArray type;
-    transaction.openDatabase("revisionType")
-        .scan(QByteArray::number(revision),
-            [&](const QByteArray &, const QByteArray &value) -> bool {
+    transaction.openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys)
+        .scan(revision,
+            [&](const size_t, const QByteArray &value) -> bool {
                 type = QByteArray{value.constData(), value.size()};
                 return false;
             },
@@ -145,17 +188,31 @@ QByteArray DataStore::getTypeFromRevision(const DataStore::Transaction &transact
     return type;
 }
 
-void DataStore::recordRevision(DataStore::Transaction &transaction, qint64 revision, const QByteArray &uid, const QByteArray &type)
+void DataStore::recordRevision(DataStore::Transaction &transaction, size_t revision,
+    const QByteArray &uid, const QByteArray &type)
 {
-    // TODO use integerkeys
-    transaction.openDatabase("revisions").write(QByteArray::number(revision), uid);
-    transaction.openDatabase("revisionType").write(QByteArray::number(revision), type);
+    transaction
+        .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys)
+        .write(revision, uid);
+    transaction.openDatabase("uidsToRevisions", /* errorHandler = */ {}, AllowDuplicates | IntegerValues)
+        .write(uid, sizeTToByteArray(revision));
+    transaction
+        .openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys)
+        .write(revision, type);
 }
 
-void DataStore::removeRevision(DataStore::Transaction &transaction, qint64 revision)
+void DataStore::removeRevision(DataStore::Transaction &transaction, size_t revision)
 {
-    transaction.openDatabase("revisions").remove(QByteArray::number(revision));
-    transaction.openDatabase("revisionType").remove(QByteArray::number(revision));
+    const QByteArray uid = getUidFromRevision(transaction, revision);
+
+    transaction
+        .openDatabase("revisions", /* errorHandler = */ {}, IntegerKeys)
+        .remove(revision);
+    transaction.openDatabase("uidsToRevisions", /* errorHandler = */ {}, AllowDuplicates | IntegerValues)
+        .remove(uid, sizeTToByteArray(revision));
+    transaction
+        .openDatabase("revisionType", /* errorHandler = */ {}, IntegerKeys)
+        .remove(revision);
 }
 
 void DataStore::recordUid(DataStore::Transaction &transaction, const QByteArray &uid, const QByteArray &type)
@@ -176,6 +233,18 @@ void DataStore::getUids(const QByteArray &type, const Transaction &transaction,
     });
 }
 
+bool DataStore::hasUid(const QByteArray &type, const Transaction &transaction, const QByteArray &uid)
+{
+    bool hasTheUid = false;
+    transaction.openDatabase(type + "uids").scan(uid, [&](const QByteArray &key, const QByteArray &) {
+        Q_ASSERT(uid == key);
+        hasTheUid = true;
+        return false;
+    });
+
+    return hasTheUid;
+}
+
 bool DataStore::isInternalKey(const char *key)
 {
     return key && strncmp(key, s_internalPrefix, s_internalPrefixSize) == 0;
@@ -207,7 +276,7 @@ DataStore::NamedDatabase DataStore::mainDatabase(const DataStore::Transaction &t
         Q_ASSERT(false);
         return {};
     }
-    return t.openDatabase(type + ".main");
+    return t.openDatabase(type + ".main", /* errorHandler= */ {}, IntegerKeys);
 }
 
 bool DataStore::NamedDatabase::contains(const QByteArray &uid)