From 46313049ac01a3007ef60bdc937442945355a38d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Nicole?= Date: Wed, 22 Aug 2018 14:16:59 +0200 Subject: Separate UIDs and Revisions in main databases Summary: - Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine) - Better support for database flags (duplicate, integer keys, integer values for now but is extensible) - Main databases' keys are now revisions - Some databases switched to integer keys databases: - Main databases - the revision to uid mapping database - the revision to entity type mapping database - Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again) - New uid to revision mapping database (`uidsToRevisions`): - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid) ~~Problems to fix before merging:~~ All Fixed! - ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~ - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~ - ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~ ~~The first problem prevents me from running benchmarks~~ Reviewers: cmollekopf Tags: #sink Differential Revision: https://phabricator.kde.org/D14974 --- tests/dbwriter.cpp | 12 +-- tests/entitystoretest.cpp | 95 +++++++++++++++++ tests/pipelinetest.cpp | 33 +++--- tests/storagetest.cpp | 260 +++++++++++++++++++++++++++++++++++++++------- 4 files changed, 339 insertions(+), 61 deletions(-) (limited to 'tests') diff --git a/tests/dbwriter.cpp b/tests/dbwriter.cpp index 3045eac..a25faec 100644 --- a/tests/dbwriter.cpp +++ b/tests/dbwriter.cpp @@ -29,14 +29,14 @@ int main(int argc, char *argv[]) qWarning() << "No valid transaction"; return -1; } - transaction.openDatabase("a", nullptr, false).write(QByteArray::number(i), "a"); - transaction.openDatabase("b", nullptr, false).write(QByteArray::number(i), "b"); - transaction.openDatabase("c", nullptr, false).write(QByteArray::number(i), "c"); - transaction.openDatabase("p", nullptr, false).write(QByteArray::number(i), "c"); - transaction.openDatabase("q", nullptr, false).write(QByteArray::number(i), "c"); + transaction.openDatabase("a", nullptr, 0).write(QByteArray::number(i), "a"); + transaction.openDatabase("b", nullptr, 0).write(QByteArray::number(i), "b"); + transaction.openDatabase("c", nullptr, 0).write(QByteArray::number(i), "c"); + transaction.openDatabase("p", nullptr, 0).write(QByteArray::number(i), "c"); + transaction.openDatabase("q", nullptr, 0).write(QByteArray::number(i), "c"); if (i > (count/2)) { for (int d = 0; d < 40; d++) { - transaction.openDatabase("db" + QByteArray::number(d), nullptr, false).write(QByteArray::number(i), "a"); + transaction.openDatabase("db" + QByteArray::number(d), nullptr, 0).write(QByteArray::number(i), "a"); } } if ((i % 1000) == 0) { diff --git a/tests/entitystoretest.cpp b/tests/entitystoretest.cpp index 03b940b..608de4c 100644 --- a/tests/entitystoretest.cpp +++ b/tests/entitystoretest.cpp @@ -18,6 +18,7 @@ private slots: void initTestCase() { Sink::AdaptorFactoryRegistry::instance().registerFactory("test"); + Sink::AdaptorFactoryRegistry::instance().registerFactory("test"); } void cleanup() @@ -29,6 +30,100 @@ private slots: { } + void testFullScan() + { + using namespace Sink; + ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")}; + Storage::EntityStore store(resourceContext, {}); + + auto mail = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + mail.setExtractedMessageId("messageid"); + mail.setExtractedSubject("boo"); + + auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + mail2.setExtractedMessageId("messageid2"); + mail2.setExtractedSubject("foo"); + + auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + mail3.setExtractedMessageId("messageid2"); + mail3.setExtractedSubject("foo"); + + store.startTransaction(Storage::DataStore::ReadWrite); + store.add("mail", mail, false); + store.add("mail", mail2, false); + store.add("mail", mail3, false); + + mail.setExtractedSubject("foo"); + + store.modify("mail", mail, QByteArrayList{}, false); + + { + const auto ids = store.fullScan("mail"); + + QCOMPARE(ids.size(), 3); + QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier()))); + QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier()))); + QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail3.identifier()))); + } + + store.remove("mail", mail3, false); + store.commitTransaction(); + + { + const auto ids = store.fullScan("mail"); + + QCOMPARE(ids.size(), 2); + QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier()))); + QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier()))); + } + } + + void testExistsAndContains() + { + + using namespace Sink; + ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")}; + Storage::EntityStore store(resourceContext, {}); + + auto mail = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + mail.setExtractedMessageId("messageid"); + mail.setExtractedSubject("boo"); + + auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + mail2.setExtractedMessageId("messageid2"); + mail2.setExtractedSubject("foo"); + + auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + mail3.setExtractedMessageId("messageid2"); + mail3.setExtractedSubject("foo"); + + auto event = ApplicationDomain::ApplicationDomainType::createEntity("res1"); + event.setExtractedUid("messageid2"); + event.setExtractedSummary("foo"); + + store.startTransaction(Storage::DataStore::ReadWrite); + store.add("mail", mail, false); + store.add("mail", mail2, false); + store.add("mail", mail3, false); + store.add("event", event, false); + + mail.setExtractedSubject("foo"); + + store.modify("mail", mail, QByteArrayList{}, false); + store.remove("mail", mail3, false); + store.commitTransaction(); + + QVERIFY(store.contains("mail", mail.identifier())); + QVERIFY(store.contains("mail", mail2.identifier())); + QVERIFY(store.contains("mail", mail3.identifier())); + QVERIFY(store.contains("event", event.identifier())); + + QVERIFY(store.exists("mail", mail.identifier())); + QVERIFY(store.exists("mail", mail2.identifier())); + QVERIFY(!store.exists("mail", mail3.identifier())); + QVERIFY(store.exists("event", event.identifier())); + } + void readAll() { using namespace Sink; diff --git a/tests/pipelinetest.cpp b/tests/pipelinetest.cpp index b41a5c2..801a9e0 100644 --- a/tests/pipelinetest.cpp +++ b/tests/pipelinetest.cpp @@ -28,26 +28,29 @@ static void removeFromDisk(const QString &name) store.removeFromDisk(); } -static QList getKeys(const QByteArray &dbEnv, const QByteArray &name) +static QList getKeys(const QByteArray &dbEnv, const QByteArray &name) { Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db = transaction.openDatabase(name, nullptr, false); - QList result; + auto db = transaction.openDatabase(name, nullptr, Sink::Storage::IntegerKeys); + QList result; db.scan("", [&](const QByteArray &key, const QByteArray &value) { - result << key; + size_t revision = Sink::byteArrayToSizeT(key); + result << Sink::Storage::Key(Sink::Storage::Identifier::fromDisplayByteArray( + Sink::Storage::DataStore::getUidFromRevision(transaction, revision)), + revision); return true; }); return result; } -static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const QByteArray &uid) +static QByteArray getEntity(const QByteArray &dbEnv, const QByteArray &name, const Sink::Storage::Key &key) { Sink::Storage::DataStore store(Sink::storageLocation(), dbEnv, Sink::Storage::DataStore::ReadOnly); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db = transaction.openDatabase(name, nullptr, false); + auto db = transaction.openDatabase(name, nullptr, Sink::Storage::IntegerKeys); QByteArray result; - db.scan(uid, [&](const QByteArray &key, const QByteArray &value) { + db.scan(key.revision().toSizeT(), [&](size_t rev, const QByteArray &value) { result = value; return true; }); @@ -251,7 +254,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); // Execute the modification @@ -264,7 +267,7 @@ private slots: key.setRevision(2); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); @@ -299,7 +302,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); @@ -322,7 +325,7 @@ private slots: key.setRevision(3); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); @@ -343,7 +346,7 @@ private slots: auto result = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(result.size(), 1); - const auto uid = Sink::Storage::Key::fromInternalByteArray(result.first()).identifier().toDisplayByteArray(); + const auto uid = result.first().identifier().toDisplayByteArray(); // Delete entity auto deleteCommand = deleteEntityCommand(uid, 1); @@ -386,7 +389,7 @@ private slots: pipeline.startTransaction(); auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - const auto uid = Sink::Storage::Key::fromInternalByteArray(keys.first()).identifier().toDisplayByteArray(); + const auto uid = keys.first().identifier().toDisplayByteArray(); { auto modifyCommand = modifyEntityCommand(createEvent(entityFbb, "summary2"), uid, 1); pipeline.modifiedEntity(modifyCommand.constData(), modifyCommand.size()); @@ -427,7 +430,7 @@ private slots: // Get uid of written entity auto keys = getKeys(instanceIdentifier(), "event.main"); QCOMPARE(keys.size(), 1); - auto key = Sink::Storage::Key::fromInternalByteArray(keys.first()); + auto key = keys.first(); const auto uid = key.identifier().toDisplayByteArray(); //Simulate local modification @@ -453,7 +456,7 @@ private slots: key.setRevision(3); // Ensure we've got the new revision with the modification - auto buffer = getEntity(instanceIdentifier(), "event.main", key.toInternalByteArray()); + auto buffer = getEntity(instanceIdentifier(), "event.main", key); QVERIFY(!buffer.isEmpty()); Sink::EntityBuffer entityBuffer(buffer.data(), buffer.size()); auto adaptor = adaptorFactory->createAdaptor(entityBuffer.entity()); diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp index 81acc13..3d583ab 100644 --- a/tests/storagetest.cpp +++ b/tests/storagetest.cpp @@ -227,7 +227,7 @@ private slots: bool gotError = false; Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("default", nullptr, false); + auto db = transaction.openDatabase("default"); db.write("key", "value"); db.write("key", "value"); @@ -250,9 +250,10 @@ private slots: { bool gotResult = false; bool gotError = false; - Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", 0x04}}}, Sink::Storage::DataStore::ReadWrite); + const int flags = Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, {dbName, {{"default", flags}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("default", nullptr, true); + auto db = transaction.openDatabase("default", nullptr, flags); db.write("key", "value1"); db.write("key", "value2"); int numValues = db.scan("key", @@ -357,7 +358,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); store.createTransaction(Sink::Storage::DataStore::ReadWrite) - .openDatabase("test", nullptr, true) + .openDatabase("test", nullptr, Sink::Storage::AllowDuplicates) .write("key1", "value1", [&](const Sink::Storage::DataStore::Error &error) { qDebug() << error.message; gotError = true; @@ -368,9 +369,10 @@ private slots: // By default we want only exact matches void testSubstringKeys() { - Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0x04}}}, Sink::Storage::DataStore::ReadWrite); + const int flags = Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", flags}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, flags); db.write("sub", "value1"); db.write("subsub", "value2"); int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); @@ -382,7 +384,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub", "value1"); db.write("subsub", "value2"); db.write("wubsub", "value3"); @@ -395,7 +397,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); db.write("sub", "value1"); db.write("subsub", "value2"); db.write("wubsub", "value3"); @@ -408,7 +410,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub_2", "value2"); db.write("sub_1", "value1"); db.write("sub_3", "value3"); @@ -429,7 +431,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, true); + auto db = transaction.openDatabase("test", nullptr, Sink::Storage::AllowDuplicates); db.write("sub1", "value1"); int numValues = db.scan("sub", [&](const QByteArray &key, const QByteArray &value) -> bool { return true; }); @@ -440,7 +442,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub1", "value1"); db.write("sub2", "value2"); db.write("wub3", "value3"); @@ -455,7 +457,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub2", "value2"); QByteArray result; db.findLatest("sub", [&](const QByteArray &key, const QByteArray &value) { result = value; }); @@ -467,7 +469,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); db.write("sub2", "value2"); db.write("wub3", "value3"); QByteArray result; @@ -478,8 +480,8 @@ private slots: static QMap baseDbs() { - return {{"revisionType", 0}, - {"revisions", 0}, + return {{"revisionType", Sink::Storage::IntegerKeys}, + {"revisions", Sink::Storage::IntegerKeys}, {"uids", 0}, {"default", 0}, {"__flagtable", 0}}; @@ -499,7 +501,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"test", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); QByteArray result; - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); const auto uid = "{c5d06a9f-1534-4c52-b8ea-415db68bdadf}"; //Ensure we can sort 1 and 10 properly (by default string comparison 10 comes before 6) const auto id = Sink::Storage::Identifier::fromDisplayByteArray(uid); @@ -523,7 +525,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results; db.findAllInRange("0002", "0004", [&](const QByteArray &key, const QByteArray &value) { results << value; }); @@ -535,7 +537,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -559,7 +561,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -571,7 +573,7 @@ private slots: { Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("test", nullptr, false); + auto db = transaction.openDatabase("test"); setupTestFindRange(db); QByteArrayList results1; @@ -601,21 +603,21 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false); + auto db = transaction.openDatabase("testTransactionVisibility"); db.write("key1", "foo"); QCOMPARE(readValue(db, "key1"), QByteArray("foo")); { auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db2 = transaction2 - .openDatabase("testTransactionVisibility", nullptr, false); + .openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray()); } transaction.commit(); { auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); auto db2 = transaction2 - .openDatabase("testTransactionVisibility", nullptr, false); + .openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); } @@ -627,16 +629,16 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"a", 0}, {"b", 0}, {"c", 0}}}, Sink::Storage::DataStore::ReadWrite); { auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - transaction.openDatabase("a", nullptr, false); - transaction.openDatabase("b", nullptr, false); - transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("a"); + transaction.openDatabase("b"); + transaction.openDatabase("c"); transaction.commit(); } auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); for (int i = 0; i < 1000; i++) { - transaction.openDatabase("a", nullptr, false); - transaction.openDatabase("b", nullptr, false); - transaction.openDatabase("c", nullptr, false); + transaction.openDatabase("a"); + transaction.openDatabase("b"); + transaction.openDatabase("c"); transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); } } @@ -662,11 +664,11 @@ private slots: // Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly); // auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly); // for (int i = 0; i < 100000; i++) { - // transaction.openDatabase("a", nullptr, false); - // transaction.openDatabase("b", nullptr, false); - // transaction.openDatabase("c", nullptr, false); - // transaction.openDatabase("p", nullptr, false); - // transaction.openDatabase("q", nullptr, false); + // transaction.openDatabase("a"); + // transaction.openDatabase("b"); + // transaction.openDatabase("c"); + // transaction.openDatabase("p"); + // transaction.openDatabase("q"); // } // }); // } @@ -733,7 +735,7 @@ private slots: Sink::Storage::DataStore store(testDataPath, {dbName, {{"testTransactionVisibility", 0}}}, Sink::Storage::DataStore::ReadWrite); auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db = transaction.openDatabase("testTransactionVisibility", nullptr, false); + auto db = transaction.openDatabase("testTransactionVisibility"); db.write("key1", "foo"); QCOMPARE(readValue(db, "key1"), QByteArray("foo")); transaction.commit(); @@ -748,12 +750,12 @@ private slots: //This transaction should open the dbi auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false); + auto db2 = transaction2.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); //This transaction should have the dbi available auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false); + auto db3 = transaction3.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); } @@ -766,20 +768,198 @@ private slots: //This transaction should open the dbi auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite); - auto db2 = transaction2.openDatabase("testTransactionVisibility", nullptr, false); + auto db2 = transaction2.openDatabase("testTransactionVisibility"); QCOMPARE(readValue(db2, "key1"), QByteArray("foo")); //This transaction should have the dbi available (creating two write transactions obviously doesn't work) //NOTE: we don't support this scenario. A write transaction must commit or abort before a read transaction opens the same database. // auto transaction3 = store.createTransaction(Sink::Storage::DataStore::ReadOnly); - // auto db3 = transaction3.openDatabase("testTransactionVisibility", nullptr, false); + // auto db3 = transaction3.openDatabase("testTransactionVisibility"); // QCOMPARE(readValue(db3, "key1"), QByteArray("foo")); //Ensure we can still open further dbis in the write transaction - auto db4 = transaction2.openDatabase("anotherDb", nullptr, false); + auto db4 = transaction2.openDatabase("anotherDb"); } } + + void testIntegerKeys() + { + const int flags = Sink::Storage::IntegerKeys; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "test", flags } } }, Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testIntegerKeys", {}, flags); + db.write(0, "value1"); + db.write(1, "value2"); + + size_t resultKey; + QByteArray result; + int numValues = db.scan(0, [&](size_t key, const QByteArray &value) -> bool { + resultKey = key; + result = value; + return true; + }); + + QCOMPARE(numValues, 1); + QCOMPARE(resultKey, 0); + QCOMPARE(result, "value1"); + + int numValues2 = db.scan(1, [&](size_t key, const QByteArray &value) -> bool { + resultKey = key; + result = value; + return true; + }); + + QCOMPARE(numValues2, 1); + QCOMPARE(resultKey, 1); + QCOMPARE(result, "value2"); + } + + void testDuplicateIntegerKeys() + { + const int flags = Sink::Storage::IntegerKeys | Sink::Storage::AllowDuplicates; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "testDuplicateIntegerKeys", flags} } }, + Sink::Storage::DataStore::ReadWrite); + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testDuplicateIntegerKeys", {}, flags); + db.write(0, "value1"); + db.write(1, "value2"); + db.write(1, "value3"); + QSet results; + int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { + results << value; + return true; + }); + + QCOMPARE(numValues, 2); + QCOMPARE(results.size(), 2); + QVERIFY(results.contains("value2")); + QVERIFY(results.contains("value3")); + } + + void testDuplicateWithIntegerValues() + { + const int flags = Sink::Storage::AllowDuplicates | Sink::Storage::IntegerValues; + Sink::Storage::DataStore store(testDataPath, + { dbName, { { "testDuplicateWithIntegerValues", flags} } }, + Sink::Storage::DataStore::ReadWrite); + + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testDuplicateWithIntegerValues", {}, flags); + + const size_t number1 = 1; + const size_t number2 = 2; + + const QByteArray number1BA = Sink::sizeTToByteArray(number1); + const QByteArray number2BA = Sink::sizeTToByteArray(number2); + + db.write(0, number1BA); + db.write(1, number2BA); + db.write(1, number1BA); + + QList results; + int numValues = db.scan(1, [&](size_t, const QByteArray &value) -> bool { + results << value; + return true; + }); + + QCOMPARE(numValues, 2); + QCOMPARE(results.size(), 2); + QCOMPARE(results[0], number1BA); + QCOMPARE(results[1], number2BA); + } + + void testIntegerKeyMultipleOf256() + { + const int flags = Sink::Storage::IntegerKeys; + Sink::Storage::DataStore store(testDataPath, + { dbName, { {"testIntegerKeyMultipleOf256", flags} } }, + Sink::Storage::DataStore::ReadWrite); + + { + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testIntegerKeyMultipleOf256", {}, flags); + + db.write(0x100, "hello"); + db.write(0x200, "hello2"); + db.write(0x42, "hello3"); + + transaction.commit(); + } + + { + auto transaction2 = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction2.openDatabase("testIntegerKeyMultipleOf256", {}, flags); + + size_t resultKey; + QByteArray resultValue; + db.scan(0x100, [&] (size_t key, const QByteArray &value) { + resultKey = key; + resultValue = value; + return false; + }); + + QCOMPARE(resultKey, 0x100); + QCOMPARE(resultValue, "hello"); + } + } + + void testIntegerProperlySorted() + { + const int flags = Sink::Storage::IntegerKeys; + Sink::Storage::DataStore store(testDataPath, + { dbName, { {"testIntegerProperlySorted", flags} } }, + Sink::Storage::DataStore::ReadWrite); + + { + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testIntegerProperlySorted", {}, flags); + + for (size_t i = 0; i < 0x100; ++i) { + db.write(i, "hello"); + } + + size_t previous = 0; + bool success = true; + db.scan("", [&] (const QByteArray &key, const QByteArray &value) { + size_t current = Sink::byteArrayToSizeT(key); + if (current < previous) { + success = false; + return false; + } + + previous = current; + return true; + }); + + QVERIFY2(success, "Integer are not properly sorted before commit"); + + transaction.commit(); + } + + { + auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite); + auto db = transaction.openDatabase("testIntegerProperlySorted", {}, flags); + + size_t previous = 0; + bool success = true; + db.scan("", [&] (const QByteArray &key, const QByteArray &value) { + size_t current = Sink::byteArrayToSizeT(key); + if (current < previous) { + success = false; + return false; + } + + previous = current; + return true; + }); + + QVERIFY2(success, "Integer are not properly sorted after commit"); + } + } + }; QTEST_MAIN(StorageTest) -- cgit v1.2.3