Separate UIDs and Revisions in main databases

Summary:
- Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine)
- Better support for database flags (duplicate, integer keys, integer values for now but is extensible)
- Main databases' keys are now revisions
- Some databases switched to integer keys databases:
  - Main databases
  - the revision to uid mapping database
  - the revision to entity type mapping database
- Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again)
- New uid to revision mapping database (`uidsToRevisions`):
  - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions
  - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid)

~~Problems to fix before merging:~~
All Fixed!

- ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~
  - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~
- ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~

~~The first problem prevents me from running benchmarks~~

Reviewers: cmollekopf

Tags: #sink

Differential Revision: https://phabricator.kde.org/D14974

4 files changed, 339 insertions, 61 deletions

diff --git a/tests/dbwriter.cpp b/tests/dbwriter.cpp
index 3045eac..a25faec 100644
--- a/tests/dbwriter.cpp
+++ b/tests/dbwriter.cpp
@@ -29,14 +29,14 @@ int main(int argc, char *argv[])
             qWarning() << "No valid transaction";
             return -1;
         }
-        transaction.openDatabase("a", nullptr, false).write(QByteArray::number(i), "a");
-        transaction.openDatabase("b", nullptr, false).write(QByteArray::number(i), "b");
-        transaction.openDatabase("c", nullptr, false).write(QByteArray::number(i), "c");
-        transaction.openDatabase("p", nullptr, false).write(QByteArray::number(i), "c");
-        transaction.openDatabase("q", nullptr, false).write(QByteArray::number(i), "c");
+        transaction.openDatabase("a", nullptr, 0).write(QByteArray::number(i), "a");
+        transaction.openDatabase("b", nullptr, 0).write(QByteArray::number(i), "b");
+        transaction.openDatabase("c", nullptr, 0).write(QByteArray::number(i), "c");
+        transaction.openDatabase("p", nullptr, 0).write(QByteArray::number(i), "c");
+        transaction.openDatabase("q", nullptr, 0).write(QByteArray::number(i), "c");
         if (i > (count/2)) {
             for (int d = 0; d < 40; d++) {
-                transaction.openDatabase("db" + QByteArray::number(d), nullptr, false).write(QByteArray::number(i), "a");
+                transaction.openDatabase("db" + QByteArray::number(d), nullptr, 0).write(QByteArray::number(i), "a");
             }
         }
         if ((i % 1000) == 0) {
diff --git a/tests/entitystoretest.cpp b/tests/entitystoretest.cpp
index 03b940b..608de4c 100644
--- a/tests/entitystoretest.cpp
+++ b/tests/entitystoretest.cpp
@@ -18,6 +18,7 @@ private slots:
     void initTestCase()
     {
         Sink::AdaptorFactoryRegistry::instance().registerFactory<Sink::ApplicationDomain::Mail, TestMailAdaptorFactory>("test");
+        Sink::AdaptorFactoryRegistry::instance().registerFactory<Sink::ApplicationDomain::Event, TestEventAdaptorFactory>("test");
     }
 
     void cleanup()
@@ -29,6 +30,100 @@ private slots:
     {
     }
 
+    void testFullScan()
+    {
+        using namespace Sink;
+        ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")};
+        Storage::EntityStore store(resourceContext, {});
+
+        auto mail = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail.setExtractedMessageId("messageid");
+        mail.setExtractedSubject("boo");
+
+        auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail2.setExtractedMessageId("messageid2");
+        mail2.setExtractedSubject("foo");
+
+        auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail3.setExtractedMessageId("messageid2");
+        mail3.setExtractedSubject("foo");
+
+        store.startTransaction(Storage::DataStore::ReadWrite);
+        store.add("mail", mail, false);
+        store.add("mail", mail2, false);
+        store.add("mail", mail3, false);
+
+        mail.setExtractedSubject("foo");
+
+        store.modify("mail", mail, QByteArrayList{}, false);
+
+        {
+            const auto ids = store.fullScan("mail");
+
+            QCOMPARE(ids.size(), 3);
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail3.identifier())));
+        }
+
+        store.remove("mail", mail3, false);
+        store.commitTransaction();
+
+        {
+            const auto ids = store.fullScan("mail");
+
+            QCOMPARE(ids.size(), 2);
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier())));
+        }
+    }
+
+    void testExistsAndContains()
+    {
+
+        using namespace Sink;
+        ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")};
+        Storage::EntityStore store(resourceContext, {});
+
+        auto mail = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail.setExtractedMessageId("messageid");
+        mail.setExtractedSubject("boo");
+
+        auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail2.setExtractedMessageId("messageid2");
+        mail2.setExtractedSubject("foo");
+
+        auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail3.setExtractedMessageId("messageid2");
+        mail3.setExtractedSubject("foo");
+
+        auto event = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Event>("res1");
+        event.setExtractedUid("messageid2");
+        event.setExtractedSummary("foo");
+
+        store.startTransaction(Storage::DataStore::ReadWrite);
+        store.add("mail", mail, false);
+        store.add("mail", mail2, false);
+        store.add("mail", mail3, false);
+        store.add("event", event, false);
+
+        mail.setExtractedSubject("foo");
+
+        store.modify("mail", mail, QByteArrayList{}, false);
+        store.remove("mail", mail3, false);
+        store.commitTransaction();
+
+        QVERIFY(store.contains("mail", mail.identifier()));
+        QVERIFY(store.contains("mail", mail2.identifier()));
+        QVERIFY(store.contains("mail", mail3.identifier()));
+        QVERIFY(store.contains("event", event.identifier()));
+
+        QVERIFY(store.exists("mail", mail.identifier()));
+        QVERIFY(store.exists("mail", mail2.identifier()));
+        QVERIFY(!store.exists("mail", mail3.identifier()));
+        QVERIFY(store.exists("event", event.identifier()));
+    }
+
     void readAll()
     {
         using namespace Sink;
diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp
index b41a5c2..801a9e0 100644
--- a/tests/pipelinetest.cpp
+++ b/tests/pipelinetest.cpp
@@ -28,26 +28,29 @@ static void removeFromDisk(const QString &name)
     store.removeFromDisk();
 }
 
-static QList<QByteArray> getKeys(const QByteArray &dbEnv, const QByteArray &name)
+static QList<Sink::Storage::Key> getKeys(const QByteArray &dbEnv, const QByteArray &name)
 {
     Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly);
     auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-    auto db = transaction.openDatabase(name, nullptr, false);
-    QList<QByteArray> result;
+    auto db = transaction.openDatabase(name, nullptr, Sink::Storage::IntegerKeys);
+    QList<Sink::Storage::Key> result;
     db.scan("", [&](const QByteArray &key, const QByteArray &value) {
-        result << key;
+        size_t revision = Sink::byteArrayToSizeT(key);
+        result << Sink::Storage::Key(Sink::Storage::Identifier::fromDisplayByteArray(
+                                         Sink::Storage::DataStore::getUidFromRevision(transaction, revision)),
+            revision);
         return true;
     });
     return result;
 }
 
-static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const QByteArray &uid)
+static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const Sink::Storage::Key &key)
 {
     Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly);
     auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-    auto db = transaction.openDatabase(name, nullptr, false);
+    auto db = transaction.openDatabase(name, nullptr, Sink::Storage::IntegerKeys);
     QByteArray result;
-    db.scan(uid, [&](const QByteArray &key, const QByteArray &value) {
+    db.scan(key.revision().toSizeT(), [&](size_t rev, const QByteArray &value) {
         result = value;
         return true;
     });
@@ -251,7 +254,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
         // Execute the modification
@@ -264,7 +267,7 @@ private slots:
         key.setRevision(2);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
@@ -299,7 +302,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
 
@@ -322,7 +325,7 @@ private slots:
         key.setRevision(3);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
@@ -343,7 +346,7 @@ private slots:
         auto result = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(result.size(), 1);
 
-        const auto uid = Sink::Storage::Key::fromInternalByteArray(result.first()).identifier().toDisplayByteArray();
+        const auto uid = result.first().identifier().toDisplayByteArray();
 
         // Delete entity
         auto deleteCommand = deleteEntityCommand(uid, 1);
@@ -386,7 +389,7 @@ private slots:
         pipeline.startTransaction();
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        const auto uid = Sink::Storage::Key::fromInternalByteArray(keys.first()).identifier().toDisplayByteArray();
+        const auto uid = keys.first().identifier().toDisplayByteArray();
         {
             auto modifyCommand = modifyEntityCommand(createEvent(entityFbb, "summary2"), uid, 1);
             pipeline.modifiedEntity(modifyCommand.constData(), modifyCommand.size());
@@ -427,7 +430,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
         //Simulate local modification
@@ -453,7 +456,7 @@ private slots:
         key.setRevision(3);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp
index 81acc13..3d583ab 100644
--- a/tests/storagetest.cpp
+++ b/tests/storagetest.cpp
@@ -227,7 +227,7 @@ private slots:
         bool gotError = false;
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("default", nullptr, false);
+        auto db = transaction.openDatabase("default");
         db.write("key", "value");
         db.write("key", "value");
 
@@ -250,9 +250,10 @@ private slots:
     {
         bool gotResult = false;
         bool gotError = false;
-        Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0x04}}}, Sink::Storage::DataStore::ReadWrite);
+        const int flags = Sink::Storage::AllowDuplicates;
+        Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", flags}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("default", nullptr, true);
+        auto db = transaction.openDatabase("default", nullptr, flags);
         db.write("key", "value1");
         db.write("key", "value2");
         int numValues = db.scan("key",
@@ -357,7 +358,7 @@ private slots:
 
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         store.createTransaction(Sink::Storage::DataStore::ReadWrite)
-            .openDatabase("test", nullptr, true)
+            .openDatabase("test", nullptr, Sink::Storage::AllowDuplicates)
             .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) {
                 qDebug() << error.message;
                 gotError = true;
@@ -368,9 +369,10 @@ private slots:
     // By default we want only exact matches
     void testSubstringKeys()
     {
-        Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0x04}}}, Sink::Storage::DataStore::ReadWrite);
+        const int flags = Sink::Storage::AllowDuplicates;
+        Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", flags}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, true);
+        auto db = transaction.openDatabase("test", nullptr, flags);
         db.write("sub", "value1");
         db.write("subsub", "value2");
         int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; });
@@ -382,7 +384,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub", "value1");
         db.write("subsub", "value2");
         db.write("wubsub", "value3");
@@ -395,7 +397,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, true);
+        auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates);
         db.write("sub", "value1");
         db.write("subsub", "value2");
         db.write("wubsub", "value3");
@@ -408,7 +410,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub_2", "value2");
         db.write("sub_1", "value1");
         db.write("sub_3", "value3");
@@ -429,7 +431,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, true);
+        auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates);
         db.write("sub1", "value1");
         int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; });
 
@@ -440,7 +442,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub1", "value1");
         db.write("sub2", "value2");
         db.write("wub3", "value3");
@@ -455,7 +457,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub2", "value2");
         QByteArray result;
         db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; });
@@ -467,7 +469,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub2", "value2");
         db.write("wub3", "value3");
         QByteArray result;
@@ -478,8 +480,8 @@ private slots:
 
     static QMap<QByteArray, int> baseDbs()
     {
-        return {{"revisionType", 0},
-                {"revisions", 0},
+        return {{"revisionType", Sink::Storage::IntegerKeys},
+                {"revisions", Sink::Storage::IntegerKeys},
                 {"uids", 0},
                 {"default", 0},
                 {"__flagtable", 0}};
@@ -499,7 +501,7 @@ private slots:
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
         QByteArray result;
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         const auto uid = "{c5d06a9f-1534-4c52-b8ea-415db68bdadf}";
         //Ensure we can sort 1 and 10 properly (by default string comparison 10 comes before 6)
         const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid);
@@ -523,7 +525,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
         QByteArrayList results;
         db.findAllInRange("0002", "0004", [&](const QByteArray &key, const QByteArray &value) { results << value; });
@@ -535,7 +537,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
 
         QByteArrayList results1;
@@ -559,7 +561,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
 
         QByteArrayList results1;
@@ -571,7 +573,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
 
         QByteArrayList results1;
@@ -601,21 +603,21 @@ private slots:
             Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite);
             auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
 
-            auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db = transaction.openDatabase("testTransactionVisibility");
             db.write("key1", "foo");
             QCOMPARE(readValue(db, "key1"), QByteArray("foo"));
 
             {
                 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
                 auto db2 = transaction2
-                    .openDatabase("testTransactionVisibility", nullptr, false);
+                    .openDatabase("testTransactionVisibility");
                 QCOMPARE(readValue(db2, "key1"), QByteArray());
             }
             transaction.commit();
             {
                 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
                 auto db2 = transaction2
-                    .openDatabase("testTransactionVisibility", nullptr, false);
+                    .openDatabase("testTransactionVisibility");
                 QCOMPARE(readValue(db2, "key1"), QByteArray("foo"));
             }
 
@@ -627,16 +629,16 @@ private slots:
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"a", 0}, {"b", 0}, {"c", 0}}}, Sink::Storage::DataStore::ReadWrite);
         {
             auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-            transaction.openDatabase("a", nullptr, false);
-            transaction.openDatabase("b", nullptr, false);
-            transaction.openDatabase("c", nullptr, false);
+            transaction.openDatabase("a");
+            transaction.openDatabase("b");
+            transaction.openDatabase("c");
             transaction.commit();
         }
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
         for (int i = 0; i < 1000; i++) {
-            transaction.openDatabase("a", nullptr, false);
-            transaction.openDatabase("b", nullptr, false);
-            transaction.openDatabase("c", nullptr, false);
+            transaction.openDatabase("a");
+            transaction.openDatabase("b");
+            transaction.openDatabase("c");
             transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
         }
     }
@@ -662,11 +664,11 @@ private slots:
     //            Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly);
     //            auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
     //            for (int i = 0; i < 100000; i++) {
-    //                transaction.openDatabase("a", nullptr, false);
-    //                transaction.openDatabase("b", nullptr, false);
-    //                transaction.openDatabase("c", nullptr, false);
-    //                transaction.openDatabase("p", nullptr, false);
-    //                transaction.openDatabase("q", nullptr, false);
+    //                transaction.openDatabase("a");
+    //                transaction.openDatabase("b");
+    //                transaction.openDatabase("c");
+    //                transaction.openDatabase("p");
+    //                transaction.openDatabase("q");
     //            }
     //        });
     //    }
@@ -733,7 +735,7 @@ private slots:
             Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite);
             auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
 
-            auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db = transaction.openDatabase("testTransactionVisibility");
             db.write("key1", "foo");
             QCOMPARE(readValue(db, "key1"), QByteArray("foo"));
             transaction.commit();
@@ -748,12 +750,12 @@ private slots:
 
             //This transaction should open the dbi
             auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-            auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db2 = transaction2.openDatabase("testTransactionVisibility");
             QCOMPARE(readValue(db2, "key1"), QByteArray("foo"));
 
             //This transaction should have the dbi available
             auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-            auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db3 = transaction3.openDatabase("testTransactionVisibility");
             QCOMPARE(readValue(db3, "key1"), QByteArray("foo"));
         }
 
@@ -766,20 +768,198 @@ private slots:
 
             //This transaction should open the dbi
             auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-            auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db2 = transaction2.openDatabase("testTransactionVisibility");
             QCOMPARE(readValue(db2, "key1"), QByteArray("foo"));
 
             //This transaction should have the dbi available (creating two write transactions obviously doesn't work)
             //NOTE: we don't support this scenario. A write transaction must commit or abort before a read transaction opens the same database.
             // auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-            // auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false);
+            // auto db3 = transaction3.openDatabase("testTransactionVisibility");
             // QCOMPARE(readValue(db3, "key1"), QByteArray("foo"));
 
             //Ensure we can still open further dbis in the write transaction
-            auto db4 = transaction2.openDatabase("anotherDb", nullptr, false);
+            auto db4 = transaction2.openDatabase("anotherDb");
         }
 
     }
+
+    void testIntegerKeys()
+    {
+        const int flags = Sink::Storage::IntegerKeys;
+        Sink::Storage::DataStore store(testDataPath,
+            { dbName, { { "test", flags } } }, Sink::Storage::DataStore::ReadWrite);
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        auto db = transaction.openDatabase("testIntegerKeys", {}, flags);
+        db.write(0, "value1");
+        db.write(1, "value2");
+
+        size_t resultKey;
+        QByteArray result;
+        int numValues = db.scan(0, [&](size_t key, const QByteArray &value) -> bool {
+            resultKey = key;
+            result = value;
+            return true;
+        });
+
+        QCOMPARE(numValues, 1);
+        QCOMPARE(resultKey, 0);
+        QCOMPARE(result, "value1");
+
+        int numValues2 = db.scan(1, [&](size_t key, const QByteArray &value) -> bool {
+            resultKey = key;
+            result = value;
+            return true;
+        });
+
+        QCOMPARE(numValues2, 1);
+        QCOMPARE(resultKey, 1);
+        QCOMPARE(result, "value2");
+    }
+
+    void testDuplicateIntegerKeys()
+    {
+        const int flags = Sink::Storage::IntegerKeys | Sink::Storage::AllowDuplicates;
+        Sink::Storage::DataStore store(testDataPath,
+            { dbName, { { "testDuplicateIntegerKeys", flags} } },
+            Sink::Storage::DataStore::ReadWrite);
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags);
+        db.write(0, "value1");
+        db.write(1, "value2");
+        db.write(1, "value3");
+        QSet<QByteArray> results;
+        int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool {
+            results << value;
+            return true;
+        });
+
+        QCOMPARE(numValues, 2);
+        QCOMPARE(results.size(), 2);
+        QVERIFY(results.contains("value2"));
+        QVERIFY(results.contains("value3"));
+    }
+
+    void testDuplicateWithIntegerValues()
+    {
+        const int flags = Sink::Storage::AllowDuplicates | Sink::Storage::IntegerValues;
+        Sink::Storage::DataStore store(testDataPath,
+            { dbName, { { "testDuplicateWithIntegerValues", flags} } },
+            Sink::Storage::DataStore::ReadWrite);
+
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags);
+
+        const size_t number1 = 1;
+        const size_t number2 = 2;
+
+        const QByteArray number1BA = Sink::sizeTToByteArray(number1);
+        const QByteArray number2BA = Sink::sizeTToByteArray(number2);
+
+        db.write(0, number1BA);
+        db.write(1, number2BA);
+        db.write(1, number1BA);
+
+        QList<QByteArray> results;
+        int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool {
+            results << value;
+            return true;
+        });
+
+        QCOMPARE(numValues, 2);
+        QCOMPARE(results.size(), 2);
+        QCOMPARE(results[0], number1BA);
+        QCOMPARE(results[1], number2BA);
+    }
+
+    void testIntegerKeyMultipleOf256()
+    {
+        const int flags = Sink::Storage::IntegerKeys;
+        Sink::Storage::DataStore store(testDataPath,
+                { dbName, { {"testIntegerKeyMultipleOf256", flags} } },
+                Sink::Storage::DataStore::ReadWrite);
+
+        {
+            auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+            auto db = transaction.openDatabase("testIntegerKeyMultipleOf256", {}, flags);
+
+            db.write(0x100, "hello");
+            db.write(0x200, "hello2");
+            db.write(0x42, "hello3");
+
+            transaction.commit();
+        }
+
+        {
+            auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+            auto db = transaction2.openDatabase("testIntegerKeyMultipleOf256", {}, flags);
+
+            size_t resultKey;
+            QByteArray resultValue;
+            db.scan(0x100, [&] (size_t key, const QByteArray &value) {
+                resultKey = key;
+                resultValue = value;
+                return false;
+            });
+
+            QCOMPARE(resultKey, 0x100);
+            QCOMPARE(resultValue, "hello");
+        }
+    }
+
+    void testIntegerProperlySorted()
+    {
+        const int flags = Sink::Storage::IntegerKeys;
+        Sink::Storage::DataStore store(testDataPath,
+                { dbName, { {"testIntegerProperlySorted", flags} } },
+                Sink::Storage::DataStore::ReadWrite);
+
+        {
+            auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+            auto db = transaction.openDatabase("testIntegerProperlySorted", {}, flags);
+
+            for (size_t i = 0; i < 0x100; ++i) {
+                db.write(i, "hello");
+            }
+
+            size_t previous = 0;
+            bool success = true;
+            db.scan("", [&] (const QByteArray &key, const QByteArray &value) {
+                size_t current = Sink::byteArrayToSizeT(key);
+                if (current < previous) {
+                    success = false;
+                    return false;
+                }
+
+                previous = current;
+                return true;
+            });
+
+            QVERIFY2(success, "Integer are not properly sorted before commit");
+
+            transaction.commit();
+        }
+
+        {
+            auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+            auto db = transaction.openDatabase("testIntegerProperlySorted", {}, flags);
+
+            size_t previous = 0;
+            bool success = true;
+            db.scan("", [&] (const QByteArray &key, const QByteArray &value) {
+                size_t current = Sink::byteArrayToSizeT(key);
+                if (current < previous) {
+                    success = false;
+                    return false;
+                }
+
+                previous = current;
+                return true;
+            });
+
+            QVERIFY2(success, "Integer are not properly sorted after commit");
+        }
+    }
+
 };
 
 QTEST_MAIN(StorageTest)