Separate UIDs and Revisions in main databases

Summary:
- Change revision type from `qint64` to `size_t` for LMDB in a couple of places (LMDB supports `unsigned int` or `size_t` which are `long unsigned int` on my machine)
- Better support for database flags (duplicate, integer keys, integer values for now but is extensible)
- Main databases' keys are now revisions
- Some databases switched to integer keys databases:
  - Main databases
  - the revision to uid mapping database
  - the revision to entity type mapping database
- Refactor the entity type's `typeDatabases` method (if in the future we need to change the main databases' flags again)
- New uid to revision mapping database (`uidsToRevisions`):
  - Stores all revisions (not uid to latest revision) because we need it for cleaning old revisions
  - Flags are: duplicates + integer values (so findLatest finds the latest revision for the given uid)

~~Problems to fix before merging:~~
All Fixed!

- ~~Sometimes Sink can't read what has just been written to the database (maybe because of transactions race conditions)~~
  - ~~Most of the times, this results in Sink not able to find the uid for a given revision by reading the `revisions` database~~
- ~~`pipelinetest`'s `testModifyWithConflict` fails because the local changes are overridden~~

~~The first problem prevents me from running benchmarks~~

Reviewers: cmollekopf

Tags: #sink

Differential Revision: https://phabricator.kde.org/D14974

1 files changed, 95 insertions, 0 deletions

diff --git a/tests/entitystoretest.cpp b/tests/entitystoretest.cpp
index 03b940b..608de4c 100644
--- a/tests/entitystoretest.cpp
+++ b/tests/entitystoretest.cpp
@@ -18,6 +18,7 @@ private slots:
     void initTestCase()
     {
         Sink::AdaptorFactoryRegistry::instance().registerFactory<Sink::ApplicationDomain::Mail, TestMailAdaptorFactory>("test");
+        Sink::AdaptorFactoryRegistry::instance().registerFactory<Sink::ApplicationDomain::Event, TestEventAdaptorFactory>("test");
     }
 
     void cleanup()
@@ -29,6 +30,100 @@ private slots:
     {
     }
 
+    void testFullScan()
+    {
+        using namespace Sink;
+        ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")};
+        Storage::EntityStore store(resourceContext, {});
+
+        auto mail = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail.setExtractedMessageId("messageid");
+        mail.setExtractedSubject("boo");
+
+        auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail2.setExtractedMessageId("messageid2");
+        mail2.setExtractedSubject("foo");
+
+        auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail3.setExtractedMessageId("messageid2");
+        mail3.setExtractedSubject("foo");
+
+        store.startTransaction(Storage::DataStore::ReadWrite);
+        store.add("mail", mail, false);
+        store.add("mail", mail2, false);
+        store.add("mail", mail3, false);
+
+        mail.setExtractedSubject("foo");
+
+        store.modify("mail", mail, QByteArrayList{}, false);
+
+        {
+            const auto ids = store.fullScan("mail");
+
+            QCOMPARE(ids.size(), 3);
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail3.identifier())));
+        }
+
+        store.remove("mail", mail3, false);
+        store.commitTransaction();
+
+        {
+            const auto ids = store.fullScan("mail");
+
+            QCOMPARE(ids.size(), 2);
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail.identifier())));
+            QVERIFY(ids.contains(Sink::Storage::Identifier::fromDisplayByteArray(mail2.identifier())));
+        }
+    }
+
+    void testExistsAndContains()
+    {
+
+        using namespace Sink;
+        ResourceContext resourceContext{resourceInstanceIdentifier.toUtf8(), "dummy", AdaptorFactoryRegistry::instance().getFactories("test")};
+        Storage::EntityStore store(resourceContext, {});
+
+        auto mail = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail.setExtractedMessageId("messageid");
+        mail.setExtractedSubject("boo");
+
+        auto mail2 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail2.setExtractedMessageId("messageid2");
+        mail2.setExtractedSubject("foo");
+
+        auto mail3 = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Mail>("res1");
+        mail3.setExtractedMessageId("messageid2");
+        mail3.setExtractedSubject("foo");
+
+        auto event = ApplicationDomain::ApplicationDomainType::createEntity<ApplicationDomain::Event>("res1");
+        event.setExtractedUid("messageid2");
+        event.setExtractedSummary("foo");
+
+        store.startTransaction(Storage::DataStore::ReadWrite);
+        store.add("mail", mail, false);
+        store.add("mail", mail2, false);
+        store.add("mail", mail3, false);
+        store.add("event", event, false);
+
+        mail.setExtractedSubject("foo");
+
+        store.modify("mail", mail, QByteArrayList{}, false);
+        store.remove("mail", mail3, false);
+        store.commitTransaction();
+
+        QVERIFY(store.contains("mail", mail.identifier()));
+        QVERIFY(store.contains("mail", mail2.identifier()));
+        QVERIFY(store.contains("mail", mail3.identifier()));
+        QVERIFY(store.contains("event", event.identifier()));
+
+        QVERIFY(store.exists("mail", mail.identifier()));
+        QVERIFY(store.exists("mail", mail2.identifier()));
+        QVERIFY(!store.exists("mail", mail3.identifier()));
+        QVERIFY(store.exists("event", event.identifier()));
+    }
+
     void readAll()
     {
         using namespace Sink;