2 files changed, 133 insertions, 53 deletions

diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp
index b41a5c2..47d443f 100644
--- a/tests/pipelinetest.cpp
+++ b/tests/pipelinetest.cpp
@@ -28,26 +28,29 @@ static void removeFromDisk(const QString &name)
     store.removeFromDisk();
 }
 
-static QList<QByteArray> getKeys(const QByteArray &dbEnv, const QByteArray &name)
+static QList<Sink::Storage::Key> getKeys(const QByteArray &dbEnv, const QByteArray &name)
 {
     Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly);
     auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
     auto db = transaction.openDatabase(name, nullptr, false);
-    QList<QByteArray> result;
+    QList<Sink::Storage::Key> result;
     db.scan("", [&](const QByteArray &key, const QByteArray &value) {
-        result << key;
+        size_t revision = *reinterpret_cast<const char *>(key.constData());
+        result << Sink::Storage::Key(Sink::Storage::Identifier::fromDisplayByteArray(
+                                         Sink::Storage::DataStore::getUidFromRevision(transaction, revision)),
+            revision);
         return true;
     });
     return result;
 }
 
-static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const QByteArray &uid)
+static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const Sink::Storage::Key &key)
 {
     Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly);
     auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
     auto db = transaction.openDatabase(name, nullptr, false);
     QByteArray result;
-    db.scan(uid, [&](const QByteArray &key, const QByteArray &value) {
+    db.scan(key.revision().toSizeT(), [&](size_t rev, const QByteArray &value) {
         result = value;
         return true;
     });
@@ -251,7 +254,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
         // Execute the modification
@@ -264,7 +267,7 @@ private slots:
         key.setRevision(2);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
@@ -299,7 +302,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
 
@@ -322,7 +325,7 @@ private slots:
         key.setRevision(3);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
@@ -343,7 +346,7 @@ private slots:
         auto result = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(result.size(), 1);
 
-        const auto uid = Sink::Storage::Key::fromInternalByteArray(result.first()).identifier().toDisplayByteArray();
+        const auto uid = result.first().identifier().toDisplayByteArray();
 
         // Delete entity
         auto deleteCommand = deleteEntityCommand(uid, 1);
@@ -386,7 +389,7 @@ private slots:
         pipeline.startTransaction();
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        const auto uid = Sink::Storage::Key::fromInternalByteArray(keys.first()).identifier().toDisplayByteArray();
+        const auto uid = keys.first().identifier().toDisplayByteArray();
         {
             auto modifyCommand = modifyEntityCommand(createEvent(entityFbb, "summary2"), uid, 1);
             pipeline.modifiedEntity(modifyCommand.constData(), modifyCommand.size());
@@ -427,7 +430,7 @@ private slots:
         // Get uid of written entity
         auto keys = getKeys(instanceIdentifier(), "event.main");
         QCOMPARE(keys.size(), 1);
-        auto key = Sink::Storage::Key::fromInternalByteArray(keys.first());
+        auto key = keys.first();
         const auto uid = key.identifier().toDisplayByteArray();
 
         //Simulate local modification
@@ -453,7 +456,7 @@ private slots:
         key.setRevision(3);
 
         // Ensure we've got the new revision with the modification
-        auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray());
+        auto buffer = getEntity(instanceIdentifier(), "event.main", key);
         QVERIFY(!buffer.isEmpty());
         Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size());
         auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity());
diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp
index 81acc13..39fd380 100644
--- a/tests/storagetest.cpp
+++ b/tests/storagetest.cpp
@@ -227,7 +227,7 @@ private slots:
         bool gotError = false;
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("default", nullptr, false);
+        auto db = transaction.openDatabase("default");
         db.write("key", "value");
         db.write("key", "value");
 
@@ -250,9 +250,10 @@ private slots:
     {
         bool gotResult = false;
         bool gotError = false;
-        Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0x04}}}, Sink::Storage::DataStore::ReadWrite);
+        const int flags = Sink::Storage::AllowDuplicates;
+        Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", flags}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("default", nullptr, true);
+        auto db = transaction.openDatabase("default", nullptr, flags);
         db.write("key", "value1");
         db.write("key", "value2");
         int numValues = db.scan("key",
@@ -357,7 +358,7 @@ private slots:
 
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         store.createTransaction(Sink::Storage::DataStore::ReadWrite)
-            .openDatabase("test", nullptr, true)
+            .openDatabase("test", nullptr, Sink::Storage::AllowDuplicates)
             .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) {
                 qDebug() << error.message;
                 gotError = true;
@@ -368,9 +369,10 @@ private slots:
     // By default we want only exact matches
     void testSubstringKeys()
     {
-        Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0x04}}}, Sink::Storage::DataStore::ReadWrite);
+        const int flags = Sink::Storage::AllowDuplicates;
+        Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", flags}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, true);
+        auto db = transaction.openDatabase("test", nullptr, flags);
         db.write("sub", "value1");
         db.write("subsub", "value2");
         int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; });
@@ -382,7 +384,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub", "value1");
         db.write("subsub", "value2");
         db.write("wubsub", "value3");
@@ -395,7 +397,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, true);
+        auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates);
         db.write("sub", "value1");
         db.write("subsub", "value2");
         db.write("wubsub", "value3");
@@ -408,7 +410,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub_2", "value2");
         db.write("sub_1", "value1");
         db.write("sub_3", "value3");
@@ -429,7 +431,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, true);
+        auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates);
         db.write("sub1", "value1");
         int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; });
 
@@ -440,7 +442,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub1", "value1");
         db.write("sub2", "value2");
         db.write("wub3", "value3");
@@ -455,7 +457,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub2", "value2");
         QByteArray result;
         db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; });
@@ -467,7 +469,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         db.write("sub2", "value2");
         db.write("wub3", "value3");
         QByteArray result;
@@ -478,8 +480,8 @@ private slots:
 
     static QMap<QByteArray, int> baseDbs()
     {
-        return {{"revisionType", 0},
-                {"revisions", 0},
+        return {{"revisionType", Sink::Storage::IntegerKeys},
+                {"revisions", Sink::Storage::IntegerKeys},
                 {"uids", 0},
                 {"default", 0},
                 {"__flagtable", 0}};
@@ -499,7 +501,7 @@ private slots:
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
         QByteArray result;
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         const auto uid = "{c5d06a9f-1534-4c52-b8ea-415db68bdadf}";
         //Ensure we can sort 1 and 10 properly (by default string comparison 10 comes before 6)
         const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid);
@@ -523,7 +525,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
         QByteArrayList results;
         db.findAllInRange("0002", "0004", [&](const QByteArray &key, const QByteArray &value) { results << value; });
@@ -535,7 +537,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
 
         QByteArrayList results1;
@@ -559,7 +561,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
 
         QByteArrayList results1;
@@ -571,7 +573,7 @@ private slots:
     {
         Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-        auto db = transaction.openDatabase("test", nullptr, false);
+        auto db = transaction.openDatabase("test");
         setupTestFindRange(db);
 
         QByteArrayList results1;
@@ -601,21 +603,21 @@ private slots:
             Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite);
             auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
 
-            auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db = transaction.openDatabase("testTransactionVisibility");
             db.write("key1", "foo");
             QCOMPARE(readValue(db, "key1"), QByteArray("foo"));
 
             {
                 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
                 auto db2 = transaction2
-                    .openDatabase("testTransactionVisibility", nullptr, false);
+                    .openDatabase("testTransactionVisibility");
                 QCOMPARE(readValue(db2, "key1"), QByteArray());
             }
             transaction.commit();
             {
                 auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
                 auto db2 = transaction2
-                    .openDatabase("testTransactionVisibility", nullptr, false);
+                    .openDatabase("testTransactionVisibility");
                 QCOMPARE(readValue(db2, "key1"), QByteArray("foo"));
             }
 
@@ -627,16 +629,16 @@ private slots:
         Sink::Storage::DataStore store(testDataPath, {dbName, {{"a", 0}, {"b", 0}, {"c", 0}}}, Sink::Storage::DataStore::ReadWrite);
         {
             auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-            transaction.openDatabase("a", nullptr, false);
-            transaction.openDatabase("b", nullptr, false);
-            transaction.openDatabase("c", nullptr, false);
+            transaction.openDatabase("a");
+            transaction.openDatabase("b");
+            transaction.openDatabase("c");
             transaction.commit();
         }
         auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
         for (int i = 0; i < 1000; i++) {
-            transaction.openDatabase("a", nullptr, false);
-            transaction.openDatabase("b", nullptr, false);
-            transaction.openDatabase("c", nullptr, false);
+            transaction.openDatabase("a");
+            transaction.openDatabase("b");
+            transaction.openDatabase("c");
             transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
         }
     }
@@ -662,11 +664,11 @@ private slots:
     //            Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly);
     //            auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
     //            for (int i = 0; i < 100000; i++) {
-    //                transaction.openDatabase("a", nullptr, false);
-    //                transaction.openDatabase("b", nullptr, false);
-    //                transaction.openDatabase("c", nullptr, false);
-    //                transaction.openDatabase("p", nullptr, false);
-    //                transaction.openDatabase("q", nullptr, false);
+    //                transaction.openDatabase("a");
+    //                transaction.openDatabase("b");
+    //                transaction.openDatabase("c");
+    //                transaction.openDatabase("p");
+    //                transaction.openDatabase("q");
     //            }
     //        });
     //    }
@@ -733,7 +735,7 @@ private slots:
             Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite);
             auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
 
-            auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db = transaction.openDatabase("testTransactionVisibility");
             db.write("key1", "foo");
             QCOMPARE(readValue(db, "key1"), QByteArray("foo"));
             transaction.commit();
@@ -748,12 +750,12 @@ private slots:
 
             //This transaction should open the dbi
             auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-            auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db2 = transaction2.openDatabase("testTransactionVisibility");
             QCOMPARE(readValue(db2, "key1"), QByteArray("foo"));
 
             //This transaction should have the dbi available
             auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-            auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db3 = transaction3.openDatabase("testTransactionVisibility");
             QCOMPARE(readValue(db3, "key1"), QByteArray("foo"));
         }
 
@@ -766,20 +768,95 @@ private slots:
 
             //This transaction should open the dbi
             auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
-            auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false);
+            auto db2 = transaction2.openDatabase("testTransactionVisibility");
             QCOMPARE(readValue(db2, "key1"), QByteArray("foo"));
 
             //This transaction should have the dbi available (creating two write transactions obviously doesn't work)
             //NOTE: we don't support this scenario. A write transaction must commit or abort before a read transaction opens the same database.
             // auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
-            // auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false);
+            // auto db3 = transaction3.openDatabase("testTransactionVisibility");
             // QCOMPARE(readValue(db3, "key1"), QByteArray("foo"));
 
             //Ensure we can still open further dbis in the write transaction
-            auto db4 = transaction2.openDatabase("anotherDb", nullptr, false);
+            auto db4 = transaction2.openDatabase("anotherDb");
         }
 
     }
+
+    void testIntegerKeys()
+    {
+        const int flags = Sink::Storage::IntegerKeys;
+        Sink::Storage::DataStore store(testDataPath,
+            { dbName, { { "test", flags } } }, Sink::Storage::DataStore::ReadWrite);
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        auto db = transaction.openDatabase("testIntegerKeys", {}, flags);
+        db.write(0, "value1");
+        db.write(1, "value2");
+        QByteArray result;
+        int numValues = db.scan(0, [&](size_t, const QByteArray &value) -> bool {
+            result = value;
+            return true;
+        });
+
+        QCOMPARE(numValues, 1);
+        QCOMPARE(result, "value1");
+    }
+
+    void testDuplicateIntegerKeys()
+    {
+        const int flags = Sink::Storage::IntegerKeys | Sink::Storage::AllowDuplicates;
+        Sink::Storage::DataStore store(testDataPath,
+            { dbName, { { "testDuplicateIntegerKeys", flags} } },
+            Sink::Storage::DataStore::ReadWrite);
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags);
+        db.write(0, "value1");
+        db.write(1, "value2");
+        db.write(1, "value3");
+        QSet<QByteArray> results;
+        int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool {
+            results << value;
+            return true;
+        });
+
+        QCOMPARE(numValues, 2);
+        QCOMPARE(results.size(), 2);
+        QVERIFY(results.contains("value2"));
+        QVERIFY(results.contains("value3"));
+    }
+
+    void testDuplicateWithIntegerValues()
+    {
+        const int flags = Sink::Storage::AllowDuplicates | Sink::Storage::IntegerValues;
+        Sink::Storage::DataStore store(testDataPath,
+            { dbName, { { "testDuplicateWithIntegerValues", flags} } },
+            Sink::Storage::DataStore::ReadWrite);
+
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags);
+
+        const size_t number1 = 1;
+        const size_t number2 = 2;
+
+        const QByteArray number1BA = QByteArray::fromRawData(reinterpret_cast<const char*>(&number1), sizeof(size_t));
+        const QByteArray number2BA = QByteArray::fromRawData(reinterpret_cast<const char*>(&number2), sizeof(size_t));
+
+        db.write(0, number1BA);
+        db.write(1, number2BA);
+        db.write(1, number1BA);
+
+        QList<QByteArray> results;
+        int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool {
+            results << value;
+            return true;
+        });
+
+        QCOMPARE(numValues, 2);
+        QCOMPARE(results.size(), 2);
+        QVERIFY(results[0] == number1BA);
+        QVERIFY(results[1] == number2BA);
+    }
+
 };
 
 QTEST_MAIN(StorageTest)