From 6ef0a29d8e468de50c9dcf260db45957d028a083 Mon Sep 17 00:00:00 2001 From: Minijackson Date: Tue, 21 Aug 2018 12:03:40 +0200 Subject: Separate UIDs and revisions --- tests/pipelinetest.cpp | 29 +++++---- tests/storagetest.cpp | 157 ++++++++++++++++++++++++++++++++++++------------- 2 files changed, 133 insertions(+), 53 deletions(-) (limited to 'tests') diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp index b41a5c2..47d443f 100644 --- a/tests/pipelinetest.cpp +++ b/tests/pipelinetest.cpp @@ -28,26 +28,29 @@ static void removeFromDisk(const QString &name) store.removeFromDisk(); } -static QList getKeys(const QByteArray &dbEnv, const QByteArray &name) +static QList getKeys(const QByteArray &dbEnv, const QByteArray &name) { Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db = transaction.openDatabase(name, nullptr, false); - QList result; + QList result; db.scan("", [&](const QByteArray &key, const QByteArray &value) { - result << key; + size_t revision = *reinterpret_cast(key.constData()); + result << Sink::Storage::Key(Sink::Storage::Identifier::fromDisplayByteArray( + Sink::Storage::DataStore::getUidFromRevision(transaction, revision)), + revision); return true; }); return result; } -static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const QByteArray &uid) +static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const Sink::Storage::Key &key) { Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db = transaction.openDatabase(name, nullptr, false); QByteArray result; - db.scan(uid, [&](const QByteArray &key, const QByteArray &value) { + db.scan(key.revision().toSizeT(), [&](size_t rev, const QByteArray &value) { result = value; return true; }); @@ -251,7 +254,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); // Execute the modification @@ -264,7 +267,7 @@ private slots: key.setRevision(2); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); @@ -299,7 +302,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); @@ -322,7 +325,7 @@ private slots: key.setRevision(3); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); @@ -343,7 +346,7 @@ private slots: auto result = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(result.size(), 1); - const auto uid = Sink::Storage::Key::fromInternalByteArray(result.first()).identifier().toDisplayByteArray(); + const auto uid = result.first().identifier().toDisplayByteArray(); // Delete entity auto deleteCommand = deleteEntityCommand(uid, 1); @@ -386,7 +389,7 @@ private slots: pipeline.startTransaction(); auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - const auto uid = Sink::Storage::Key::fromInternalByteArray(keys.first()).identifier().toDisplayByteArray(); + const auto uid = keys.first().identifier().toDisplayByteArray(); { auto modifyCommand = modifyEntityCommand(createEvent(entityFbb, "summary2"), uid, 1); pipeline.modifiedEntity(modifyCommand.constData(), modifyCommand.size()); @@ -427,7 +430,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); //Simulate local modification @@ -453,7 +456,7 @@ private slots: key.setRevision(3); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index 81acc13..39fd380 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp @@ -227,7 +227,7 @@ private slots: bool gotError = false; Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("default", nullptr, false); + auto db = transaction.openDatabase("default"); db.write("key", "value"); db.write("key", "value"); @@ -250,9 +250,10 @@ private slots: { bool gotResult = false; bool gotError = false; - Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0x04}}}, Sink::Storage::DataStore::ReadWrite); + const int flags = Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", flags}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("default", nullptr, true); + auto db = transaction.openDatabase("default", nullptr, flags); db.write("key", "value1"); db.write("key", "value2"); int numValues = db.scan("key", @@ -357,7 +358,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); store.createTransaction(Sink::Storage::DataStore::ReadWrite) - .openDatabase("test", nullptr, true) + .openDatabase("test", nullptr, Sink::Storage::AllowDuplicates) .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) { qDebug() << error.message; gotError = true; @@ -368,9 +369,10 @@ private slots: // By default we want only exact matches void testSubstringKeys() { - Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0x04}}}, Sink::Storage::DataStore::ReadWrite); + const int flags = Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", flags}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, flags); db.write("sub", "value1"); db.write("subsub", "value2"); int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); @@ -382,7 +384,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub", "value1"); db.write("subsub", "value2"); db.write("wubsub", "value3"); @@ -395,7 +397,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); db.write("sub", "value1"); db.write("subsub", "value2"); db.write("wubsub", "value3"); @@ -408,7 +410,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub_2", "value2"); db.write("sub_1", "value1"); db.write("sub_3", "value3"); @@ -429,7 +431,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); db.write("sub1", "value1"); int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); @@ -440,7 +442,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub1", "value1"); db.write("sub2", "value2"); db.write("wub3", "value3"); @@ -455,7 +457,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub2", "value2"); QByteArray result; db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; }); @@ -467,7 +469,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub2", "value2"); db.write("wub3", "value3"); QByteArray result; @@ -478,8 +480,8 @@ private slots: static QMap baseDbs() { - return {{"revisionType", 0}, - {"revisions", 0}, + return {{"revisionType", Sink::Storage::IntegerKeys}, + {"revisions", Sink::Storage::IntegerKeys}, {"uids", 0}, {"default", 0}, {"__flagtable", 0}}; @@ -499,7 +501,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); QByteArray result; - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); const auto uid = "{c5d06a9f-1534-4c52-b8ea-415db68bdadf}"; //Ensure we can sort 1 and 10 properly (by default string comparison 10 comes before 6) const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid); @@ -523,7 +525,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results; db.findAllInRange("0002", "0004", [&](const QByteArray &key, const QByteArray &value) { results << value; }); @@ -535,7 +537,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -559,7 +561,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -571,7 +573,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -601,21 +603,21 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false); + auto db = transaction.openDatabase("testTransactionVisibility"); db.write("key1", "foo"); QCOMPARE(readValue(db, "key1"), QByteArray("foo")); { auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db2 = transaction2 - .openDatabase("testTransactionVisibility", nullptr, false); + .openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray()); } transaction.commit(); { auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db2 = transaction2 - .openDatabase("testTransactionVisibility", nullptr, false); + .openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); } @@ -627,16 +629,16 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"a", 0}, {"b", 0}, {"c", 0}}}, Sink::Storage::DataStore::ReadWrite); { auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - transaction.openDatabase("a", nullptr, false); - transaction.openDatabase("b", nullptr, false); - transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("a"); + transaction.openDatabase("b"); + transaction.openDatabase("c"); transaction.commit(); } auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); for (int i = 0; i < 1000; i++) { - transaction.openDatabase("a", nullptr, false); - transaction.openDatabase("b", nullptr, false); - transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("a"); + transaction.openDatabase("b"); + transaction.openDatabase("c"); transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); } } @@ -662,11 +664,11 @@ private slots: // Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly); // auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); // for (int i = 0; i < 100000; i++) { - // transaction.openDatabase("a", nullptr, false); - // transaction.openDatabase("b", nullptr, false); - // transaction.openDatabase("c", nullptr, false); - // transaction.openDatabase("p", nullptr, false); - // transaction.openDatabase("q", nullptr, false); + // transaction.openDatabase("a"); + // transaction.openDatabase("b"); + // transaction.openDatabase("c"); + // transaction.openDatabase("p"); + // transaction.openDatabase("q"); // } // }); // } @@ -733,7 +735,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false); + auto db = transaction.openDatabase("testTransactionVisibility"); db.write("key1", "foo"); QCOMPARE(readValue(db, "key1"), QByteArray("foo")); transaction.commit(); @@ -748,12 +750,12 @@ private slots: //This transaction should open the dbi auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false); + auto db2 = transaction2.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); //This transaction should have the dbi available auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false); + auto db3 = transaction3.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); } @@ -766,20 +768,95 @@ private slots: //This transaction should open the dbi auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false); + auto db2 = transaction2.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); //This transaction should have the dbi available (creating two write transactions obviously doesn't work) //NOTE: we don't support this scenario. A write transaction must commit or abort before a read transaction opens the same database. // auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - // auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false); + // auto db3 = transaction3.openDatabase("testTransactionVisibility"); // QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); //Ensure we can still open further dbis in the write transaction - auto db4 = transaction2.openDatabase("anotherDb", nullptr, false); + auto db4 = transaction2.openDatabase("anotherDb"); } } + + void testIntegerKeys() + { + const int flags = Sink::Storage::IntegerKeys; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "test", flags } } }, Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testIntegerKeys", {}, flags); + db.write(0, "value1"); + db.write(1, "value2"); + QByteArray result; + int numValues = db.scan(0, [&](size_t, const QByteArray &value) -> bool { + result = value; + return true; + }); + + QCOMPARE(numValues, 1); + QCOMPARE(result, "value1"); + } + + void testDuplicateIntegerKeys() + { + const int flags = Sink::Storage::IntegerKeys | Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "testDuplicateIntegerKeys", flags} } }, + Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags); + db.write(0, "value1"); + db.write(1, "value2"); + db.write(1, "value3"); + QSet results; + int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { + results << value; + return true; + }); + + QCOMPARE(numValues, 2); + QCOMPARE(results.size(), 2); + QVERIFY(results.contains("value2")); + QVERIFY(results.contains("value3")); + } + + void testDuplicateWithIntegerValues() + { + const int flags = Sink::Storage::AllowDuplicates | Sink::Storage::IntegerValues; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "testDuplicateWithIntegerValues", flags} } }, + Sink::Storage::DataStore::ReadWrite); + + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags); + + const size_t number1 = 1; + const size_t number2 = 2; + + const QByteArray number1BA = QByteArray::fromRawData(reinterpret_cast(&number1), sizeof(size_t)); + const QByteArray number2BA = QByteArray::fromRawData(reinterpret_cast(&number2), sizeof(size_t)); + + db.write(0, number1BA); + db.write(1, number2BA); + db.write(1, number1BA); + + QList results; + int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { + results << value; + return true; + }); + + QCOMPARE(numValues, 2); + QCOMPARE(results.size(), 2); + QVERIFY(results[0] == number1BA); + QVERIFY(results[1] == number2BA); + } + }; QTEST_MAIN(StorageTest) -- cgit v1.2.3