Separate UIDs and Revisions in main databases

Summary:
- Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine)
- Better support for database flags (duplicate, integer keys, integer values for now but is extensible)
- Main databases' keys are now revisions
- Some databases switched to integer keys databases:
  - Main databases
  - the revision to uid mapping database
  - the revision to entity type mapping database
- Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again)
- New uid to revision mapping database (`uidsToRevisions`):
  - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions
  - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid)

~~Problems to fix before merging:~~
All Fixed!

- ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~
  - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~
- ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~

~~The first problem prevents me from running benchmarks~~

Reviewers: cmollekopf

Tags: #sink

Differential Revision: https://phabricator.kde.org/D14974

1 files changed, 18 insertions, 15 deletions

diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp
index b41a5c2..801a9e0 100644
--- a/tests/pipelinetest.cpp
+++ b/tests/pipelinetest.cpp
@@ -28,26 +28,29 @@ static void removeFromDisk(const QString &name)
     store.removeFromDisk();
 }
 
-static QList<QByteArray> getKeys(const QByteArray &dbEnv, const QByteArray &name)
+static QList<Sink::Storage::Key> getKeys(const QByteArray &dbEnv, const QByteArray &name)
 {
     Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly);
     auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-    auto db = transaction.openDatabase(name, nullptr, false);
-    QList<QByteArray> result;
+    auto db = transaction.openDatabase(name, nullptr, Sink::Storage::IntegerKeys);
+    QList<Sink::Storage::Key> result;
     db.scan("", [&](const QByteArray &key, const QByteArray &value) {
-        result << key;
+        size_t revision = Sink::byteArrayToSizeT(key);
+        result << Sink::Storage::Key(Sink::Storage::Identifier::fromDisplayByteArray(
+                                         Sink::Storage::DataStore::getUidFromRevision(transaction, revision)),
+            revision);
         return true;
     });
     return result;
 }
 
-static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const QByteArray &uid)
+static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const Sink::Storage::Key &key)
 {
     Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly);
     auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-    auto db = transaction.openDatabase(name, nullptr, false);
+    auto db = transaction.openDatabase(name, nullptr, Sink::Storage::IntegerKeys);
     QByteArray result;
-    db.scan(uid, [&](const QByteArray &key, const QByteArray &value) {
+    db.scan(key.revision().toSizeT(), [&](size_t rev, const QByteArray &value) {
         result = value;
         return true;
     });
@@ -251,7 +254,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
         // Execute the modification
@@ -264,7 +267,7 @@ private slots:
         key.setRevision(2);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
@@ -299,7 +302,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
 
@@ -322,7 +325,7 @@ private slots:
         key.setRevision(3);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
@@ -343,7 +346,7 @@ private slots:
         auto result = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(result.size(), 1);
 
-        const auto uid = Sink::Storage::Key::fromInternalByteArray(result.first()).identifier().toDisplayByteArray();
+        const auto uid = result.first().identifier().toDisplayByteArray();
 
         // Delete entity
         auto deleteCommand = deleteEntityCommand(uid, 1);
@@ -386,7 +389,7 @@ private slots:
         pipeline.startTransaction();
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        const auto uid = Sink::Storage::Key::fromInternalByteArray(keys.first()).identifier().toDisplayByteArray();
+        const auto uid = keys.first().identifier().toDisplayByteArray();
         {
             auto modifyCommand = modifyEntityCommand(createEvent(entityFbb, "summary2"), uid, 1);
             pipeline.modifiedEntity(modifyCommand.constData(), modifyCommand.size());
@@ -427,7 +430,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
         //Simulate local modification
@@ -453,7 +456,7 @@ private slots:
         key.setRevision(3);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());