14 files changed, 306 insertions, 114 deletions

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index c77a736..2b3e7b1 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -16,6 +16,9 @@ add_library(sink_test SHARED testimplementations.cpp getrssusage.cpp mailtest.cp
 qt5_use_modules(sink_test Core Test Concurrent)
 target_link_libraries(sink_test sink libhawd KF5::Mime)
 
+add_executable(dbwriter dbwriter.cpp)
+target_link_libraries(dbwriter sink)
+
 include(SinkTest)
 
 manual_tests (
diff --git a/tests/dbwriter.cpp b/tests/dbwriter.cpp
new file mode 100644
index 0000000..902a607
--- /dev/null
+++ b/tests/dbwriter.cpp
@@ -0,0 +1,45 @@
+#include <QByteArrayList>
+#include <QDebug>
+#include <storage.h>
+
+int main(int argc, char *argv[])
+{
+
+    QByteArrayList arguments;
+    for (int i = 0; i < argc; i++) {
+        arguments << argv[i];
+    }
+    auto testDataPath = arguments.value(1);
+    auto dbName = arguments.value(2);
+    auto count = arguments.value(3).toInt();
+
+    if (Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly).exists()) {
+        Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite).removeFromDisk();
+    }
+
+    qWarning() << "Creating db: " << testDataPath << dbName << count;
+    Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
+    auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+    for (int i = 0; i < count; i++) {
+        if (!transaction) {
+            qWarning() << "No valid transaction";
+            return -1;
+        }
+        transaction.openDatabase("a", nullptr, false).write(QByteArray::number(i), "a");
+        transaction.openDatabase("b", nullptr, false).write(QByteArray::number(i), "b");
+        transaction.openDatabase("c", nullptr, false).write(QByteArray::number(i), "c");
+        transaction.openDatabase("p", nullptr, false).write(QByteArray::number(i), "c");
+        transaction.openDatabase("q", nullptr, false).write(QByteArray::number(i), "c");
+        if (i > (count/2)) {
+            for (int d = 0; d < 40; d++) {
+                transaction.openDatabase("db" + QByteArray::number(d), nullptr, false).write(QByteArray::number(i), "a");
+            }
+        }
+        if ((i % 1000) == 0) {
+            transaction.commit();
+            transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+        }
+    }
+    qWarning() << "Creating db done.";
+    return 0;
+}
diff --git a/tests/domainadaptortest.cpp b/tests/domainadaptortest.cpp
index a17152e..2aed0a9 100644
--- a/tests/domainadaptortest.cpp
+++ b/tests/domainadaptortest.cpp
@@ -14,24 +14,22 @@
 #include "metadata_generated.h"
 #include "entity_generated.h"
 
-class TestFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Event, Sink::ApplicationDomain::Buffer::Event, Sink::ApplicationDomain::Buffer::EventBuilder>
+class TestFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Event>
 {
 public:
-    TestFactory()
-    {
-        mResourceWriteMapper = QSharedPointer<WritePropertyMapper<Sink::ApplicationDomain::Buffer::EventBuilder>>::create();
-        Sink::ApplicationDomain::TypeImplementation<Sink::ApplicationDomain::Event>::configure(*mResourceWriteMapper);
-    }
+    TestFactory() = default;
 };
 
-class TestMailFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Mail, Sink::ApplicationDomain::Buffer::Mail, Sink::ApplicationDomain::Buffer::MailBuilder>
+class TestMailFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Mail>
 {
 public:
-    TestMailFactory()
-    {
-        mResourceWriteMapper = QSharedPointer<WritePropertyMapper<Sink::ApplicationDomain::Buffer::MailBuilder>>::create();
-        Sink::ApplicationDomain::TypeImplementation<Sink::ApplicationDomain::Mail>::configure(*mResourceWriteMapper);
-    }
+    TestMailFactory() = default;
+};
+
+class TestContactFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Contact>
+{
+public:
+    TestContactFactory() = default;
 };
 
 /**
@@ -51,7 +49,7 @@ private slots:
 
     void testCreateBufferPart()
     {
-        auto writeMapper = QSharedPointer<WritePropertyMapper<Sink::ApplicationDomain::Buffer::EventBuilder>>::create();
+        auto writeMapper = QSharedPointer<PropertyMapper>::create();
         Sink::ApplicationDomain::TypeImplementation<Sink::ApplicationDomain::Event>::configure(*writeMapper);
 
         Sink::ApplicationDomain::Event event;
@@ -104,7 +102,7 @@ private slots:
 
     void testMail()
     {
-        auto writeMapper = QSharedPointer<WritePropertyMapper<Sink::ApplicationDomain::Buffer::MailBuilder>>::create();
+        auto writeMapper = QSharedPointer<PropertyMapper>::create();
         Sink::ApplicationDomain::TypeImplementation<Sink::ApplicationDomain::Mail>::configure(*writeMapper);
 
         Sink::ApplicationDomain::Mail mail;
@@ -139,6 +137,43 @@ private slots:
         }
 
     }
+
+    void testContact()
+    {
+        auto writeMapper = QSharedPointer<PropertyMapper>::create();
+        Sink::ApplicationDomain::TypeImplementation<Sink::ApplicationDomain::Contact>::configure(*writeMapper);
+
+        auto binaryData = QByteArray::fromRawData("\xEF\xBF\xBD\x00\xEF\xBF", 5);
+
+        Sink::ApplicationDomain::Contact contact;
+        contact.setPhoto(binaryData);
+        QVERIFY(!contact.getPhoto().isEmpty());
+
+        flatbuffers::FlatBufferBuilder metadataFbb;
+        auto metadataBuilder = Sink::MetadataBuilder(metadataFbb);
+        metadataBuilder.add_revision(1);
+        auto metadataBuffer = metadataBuilder.Finish();
+        Sink::FinishMetadataBuffer(metadataFbb, metadataBuffer);
+
+        flatbuffers::FlatBufferBuilder mailFbb;
+        auto pos = createBufferPart<Sink::ApplicationDomain::Buffer::ContactBuilder, Sink::ApplicationDomain::Buffer::Contact>(contact, mailFbb, *writeMapper);
+        Sink::ApplicationDomain::Buffer::FinishContactBuffer(mailFbb, pos);
+
+        flatbuffers::FlatBufferBuilder fbb;
+        Sink::EntityBuffer::assembleEntityBuffer(
+            fbb, metadataFbb.GetBufferPointer(), metadataFbb.GetSize(), mailFbb.GetBufferPointer(), mailFbb.GetSize(), mailFbb.GetBufferPointer(), mailFbb.GetSize());
+
+        {
+            std::string data(reinterpret_cast<const char *>(fbb.GetBufferPointer()), fbb.GetSize());
+            Sink::EntityBuffer buffer((void *)(data.data()), data.size());
+
+            TestContactFactory factory;
+            auto adaptor = factory.createAdaptor(buffer.entity());
+            Sink::ApplicationDomain::Contact readContact{QByteArray{}, QByteArray{}, 0, adaptor};
+            QCOMPARE(readContact.getPhoto(), contact.getPhoto());
+        }
+
+    }
 };
 
 QTEST_MAIN(DomainAdaptorTest)
diff --git a/tests/hawd/state.cpp b/tests/hawd/state.cpp
index dfeef41..a994718 100644
--- a/tests/hawd/state.cpp
+++ b/tests/hawd/state.cpp
@@ -117,7 +117,7 @@ void State::findGitHash()
 {
 #ifdef HAVE_LIBGIT2
     git_libgit2_init();
-    git_buf root = {0};
+    git_buf root = GIT_BUF_INIT_CONST(0, 0);
     int error = git_repository_discover(&root, projectPath().toStdString().data(), 0, NULL);
     if (!error) {
         git_repository *repo = NULL;
diff --git a/tests/interresourcemovetest.cpp b/tests/interresourcemovetest.cpp
index 3ac6ad4..5438bc7 100644
--- a/tests/interresourcemovetest.cpp
+++ b/tests/interresourcemovetest.cpp
@@ -48,7 +48,7 @@ class InterResourceMoveTest : public QObject
         m.subject(true)->fromUnicodeString(subject, "utf8");
         m.messageID(true)->setIdentifier(uid);
         m.assemble();
-        return m.encodedContent();
+        return m.encodedContent(true);
     }
 
 private slots:
diff --git a/tests/mailquerybenchmark.cpp b/tests/mailquerybenchmark.cpp
index b15c8d6..00c156d 100644
--- a/tests/mailquerybenchmark.cpp
+++ b/tests/mailquerybenchmark.cpp
@@ -78,7 +78,7 @@ class MailQueryBenchmark : public QObject
         entityStore.commitTransaction();
     }
 
-    void testLoad(const Sink::Query &query, int count, int expectedSize)
+    void testLoad(const QByteArray &name, const Sink::Query &query, int count, int expectedSize)
     {
         const auto startingRss = getCurrentRSS();
 
@@ -124,7 +124,7 @@ class MailQueryBenchmark : public QObject
         std::cout << "Rss without db [kb]: " << rssWithoutDb / 1024 << std::endl;
         std::cout << "Percentage error: " << percentageRssError << std::endl;
 
-        HAWD::Dataset dataset("mail_query", mHawdState);
+        HAWD::Dataset dataset(QString{"mail_query"} + name, mHawdState);
         HAWD::Dataset::Row row = dataset.row();
         row.setValue("rows", list.size());
         row.setValue("queryResultPerMs", (qreal)list.size() / elapsed);
@@ -159,7 +159,7 @@ private slots:
         query.limit(1000);
 
         populateDatabase(50000);
-        testLoad(query, 50000, query.limit());
+        testLoad({}, query, 50000, query.limit());
     }
 
     void test50kThreadleader()
@@ -176,7 +176,7 @@ private slots:
 
         int count = 50000;
         populateDatabase(count, mailsPerFolder);
-        testLoad(query, count, query.limit());
+        testLoad("_threadleader", query, count, query.limit());
     }
 };
 
diff --git a/tests/mailsynctest.cpp b/tests/mailsynctest.cpp
index c8ba9f1..75454c0 100644
--- a/tests/mailsynctest.cpp
+++ b/tests/mailsynctest.cpp
@@ -33,8 +33,6 @@
 using namespace Sink;
 using namespace Sink::ApplicationDomain;
 
-SINK_DEBUG_AREA("mailsynctest")
-
 void MailSyncTest::initTestCase()
 {
     Test::initTest();
@@ -226,7 +224,8 @@ void MailSyncTest::testListNewSubFolder()
         for (const auto &folder : folders) {
             names << folder->getName();
         }
-        QVERIFY(names.contains("sub1"));
+        ASYNCVERIFY(names.contains("sub1"));
+        return KAsync::null();
     });
     VERIFYEXEC(job);
 }
@@ -254,7 +253,8 @@ void MailSyncTest::testListRemovedSubFolder()
         for (const auto &folder : folders) {
             names << folder->getName();
         }
-        QVERIFY(!names.contains("sub1"));
+        ASYNCVERIFY(!names.contains("sub1"));
+        return KAsync::null();
     });
     VERIFYEXEC(job);
 }
@@ -270,18 +270,19 @@ void MailSyncTest::testListMails()
     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
 
     auto job = Store::fetchAll<Mail>(query).then([](const QList<Mail::Ptr> &mails) {
-        QCOMPARE(mails.size(), 1);
+        ASYNCCOMPARE(mails.size(), 1);
         auto mail = mails.first();
-        QVERIFY(mail->getSubject().startsWith(QString("[Nepomuk] Jenkins build is still unstable")));
+        ASYNCVERIFY(mail->getSubject().startsWith(QString("[Nepomuk] Jenkins build is still unstable")));
         const auto data = mail->getMimeMessage();
-        QVERIFY(!data.isEmpty());
+        ASYNCVERIFY(!data.isEmpty());
 
         KMime::Message m;
-        m.setContent(data);
+        m.setContent(KMime::CRLFtoLF(data));
         m.parse();
-        QCOMPARE(mail->getSubject(), m.subject(true)->asUnicodeString());
-        QVERIFY(!mail->getFolder().isEmpty());
-        QVERIFY(mail->getDate().isValid());
+        ASYNCCOMPARE(mail->getSubject(), m.subject(true)->asUnicodeString());
+        ASYNCVERIFY(!mail->getFolder().isEmpty());
+        ASYNCVERIFY(mail->getDate().isValid());
+        return KAsync::null();
     });
     VERIFYEXEC(job);
 }
@@ -302,10 +303,11 @@ void MailSyncTest::testResyncMails()
     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
 
     auto job = Store::fetchAll<Mail>(query).then([](const QList<Mail::Ptr> &mails) {
-        QCOMPARE(mails.size(), 1);
+        ASYNCCOMPARE(mails.size(), 1);
         auto mail = mails.first();
-        QVERIFY(!mail->getSubject().isEmpty());
-        QVERIFY(!mail->getMimeMessagePath().isEmpty());
+        ASYNCVERIFY(!mail->getSubject().isEmpty());
+        ASYNCVERIFY(!mail->getMimeMessagePath().isEmpty());
+        return KAsync::null();
     });
     VERIFYEXEC(job);
 }
@@ -330,7 +332,8 @@ void MailSyncTest::testFetchNewRemovedMessages()
 
     {
         auto job = Store::fetchAll<Mail>(query).then([](const QList<Mail::Ptr> &mails) {
-            QCOMPARE(mails.size(), 2);
+            ASYNCCOMPARE(mails.size(), 2);
+            return KAsync::null();
         });
         VERIFYEXEC(job);
     }
@@ -342,7 +345,8 @@ void MailSyncTest::testFetchNewRemovedMessages()
 
     {
         auto job = Store::fetchAll<Mail>(query).then([](const QList<Mail::Ptr> &mails) {
-            QCOMPARE(mails.size(), 1);
+            ASYNCCOMPARE(mails.size(), 1);
+            return KAsync::null();
         });
         VERIFYEXEC(job);
     }
@@ -365,7 +369,8 @@ void MailSyncTest::testFlagChange()
 
     {
         auto job = Store::fetchAll<Mail>(query).then([](const QList<Mail::Ptr> &mails) {
-            QCOMPARE(mails.size(), 0);
+            ASYNCCOMPARE(mails.size(), 0);
+            return KAsync::null();
         });
         VERIFYEXEC(job);
     }
@@ -378,8 +383,9 @@ void MailSyncTest::testFlagChange()
 
     {
         auto job = Store::fetchAll<Mail>(query).then([](const QList<Mail::Ptr> &mails) {
-            QCOMPARE(mails.size(), 1);
-            QVERIFY(mails.first()->getImportant());
+            ASYNCCOMPARE(mails.size(), 1);
+            ASYNCVERIFY(mails.first()->getImportant());
+            return KAsync::null();
         });
         VERIFYEXEC(job);
     }
@@ -394,8 +400,9 @@ void MailSyncTest::testSyncSingleFolder()
     Folder::Ptr folder;
     {
         auto job = Store::fetchAll<Folder>(Sink::Query{}.resourceFilter(mResourceInstanceIdentifier).filter<Folder::Name>("test")).template then([&](const QList<Folder::Ptr> &folders) {
-            QCOMPARE(folders.size(), 1);
+            ASYNCCOMPARE(folders.size(), 1);
             folder = folders.first();
+            return KAsync::null();
         });
         VERIFYEXEC(job);
     }
@@ -419,11 +426,13 @@ void MailSyncTest::testSyncSingleMail()
     Mail::Ptr mail;
     {
         auto job = Store::fetchAll<Mail>(Sink::Query{}.resourceFilter(mResourceInstanceIdentifier)).template then([&](const QList<Mail::Ptr> &mails) {
-            QVERIFY(mails.size() >= 1);
+            ASYNCVERIFY(mails.size() >= 1);
             mail = mails.first();
+            return KAsync::null();
         });
         VERIFYEXEC(job);
     }
+    QVERIFY(mail);
 
     auto syncScope = Sink::SyncScope{ApplicationDomain::getTypeName<Mail>()};
     syncScope.resourceFilter(mResourceInstanceIdentifier);
diff --git a/tests/mailtest.cpp b/tests/mailtest.cpp
index c51fc56..cbb56d5 100644
--- a/tests/mailtest.cpp
+++ b/tests/mailtest.cpp
@@ -31,8 +31,6 @@
 using namespace Sink;
 using namespace Sink::ApplicationDomain;
 
-SINK_DEBUG_AREA("mailtest")
-
 void MailTest::initTestCase()
 {
     Test::initTest();
@@ -154,7 +152,7 @@ void MailTest::testCreateModifyDeleteMail()
     message->assemble();
 
     auto mail = Mail::create(mResourceInstanceIdentifier);
-    mail.setMimeMessage(message->encodedContent());
+    mail.setMimeMessage(message->encodedContent(true));
     mail.setFolder(folder);
 
     VERIFYEXEC(Store::create(mail));
@@ -168,7 +166,7 @@ void MailTest::testCreateModifyDeleteMail()
                 QCOMPARE(mail.getFolder(), folder.identifier());
                 QVERIFY(QFile(mail.getMimeMessagePath()).exists());
                 KMime::Message m;
-                m.setContent(mail.getMimeMessage());
+                m.setContent(KMime::CRLFtoLF(mail.getMimeMessage()));
                 m.parse();
                 QCOMPARE(m.subject(true)->asUnicodeString(), subject);
             });
@@ -184,7 +182,7 @@ void MailTest::testCreateModifyDeleteMail()
     auto message2 = KMime::Message::Ptr::create();
     message2->subject(true)->fromUnicodeString(subject2, "utf8");
     message2->assemble();
-    mail.setMimeMessage(message2->encodedContent());
+    mail.setMimeMessage(message2->encodedContent(true));
 
     VERIFYEXEC(Store::modify(mail));
     VERIFYEXEC(ResourceControl::flushMessageQueue(QByteArrayList() << mResourceInstanceIdentifier));
@@ -197,7 +195,7 @@ void MailTest::testCreateModifyDeleteMail()
                 QCOMPARE(mail.getFolder(), folder.identifier());
                 QVERIFY(QFile(mail.getMimeMessagePath()).exists());
                 KMime::Message m;
-                m.setContent(mail.getMimeMessage());
+                m.setContent(KMime::CRLFtoLF(mail.getMimeMessage()));
                 m.parse();
                 QCOMPARE(m.subject(true)->asUnicodeString(), subject2);
             });
@@ -239,7 +237,7 @@ void MailTest::testMoveMail()
     message->assemble();
 
     auto mail = Mail::create(mResourceInstanceIdentifier);
-    mail.setMimeMessage(message->encodedContent());
+    mail.setMimeMessage(message->encodedContent(true));
     mail.setFolder(folder);
 
     VERIFYEXEC(Store::create(mail));
@@ -291,7 +289,7 @@ void MailTest::testMarkMailAsRead()
     message->assemble();
 
     auto mail = Mail::create(mResourceInstanceIdentifier);
-    mail.setMimeMessage(message->encodedContent());
+    mail.setMimeMessage(message->encodedContent(true));
     mail.setFolder(folder);
     mail.setUnread(true);
     VERIFYEXEC(Store::create(mail));
@@ -343,7 +341,7 @@ void MailTest::testCreateDraft()
     message->assemble();
 
     auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier);
-    mail.setMimeMessage(message->encodedContent());
+    mail.setMimeMessage(message->encodedContent(true));
     mail.setDraft(true);
 
     VERIFYEXEC(Store::create(mail));
@@ -393,7 +391,7 @@ void MailTest::testModifyMailToDraft()
     message->assemble();
 
     auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier);
-    mail.setMimeMessage(message->encodedContent());
+    mail.setMimeMessage(message->encodedContent(true));
     mail.setDraft(false);
     mail.setFolder(folder);
 
@@ -440,7 +438,7 @@ void MailTest::testModifyMailToTrash()
     message->assemble();
 
     auto mail = ApplicationDomain::Mail::create(mResourceInstanceIdentifier);
-    mail.setMimeMessage(message->encodedContent());
+    mail.setMimeMessage(message->encodedContent(true));
     mail.setTrash(false);
     mail.setFolder(folder);
 
diff --git a/tests/mailthreadtest.cpp b/tests/mailthreadtest.cpp
index 02b29f6..741eb78 100644
--- a/tests/mailthreadtest.cpp
+++ b/tests/mailthreadtest.cpp
@@ -117,7 +117,7 @@ void MailThreadTest::testIndexInMixedOrder()
 
     {
         auto mail = Mail::create(mResourceInstanceIdentifier);
-        mail.setMimeMessage(message3->encodedContent());
+        mail.setMimeMessage(message3->encodedContent(true));
         mail.setFolder(folder);
         VERIFYEXEC(Store::create(mail));
     }
@@ -140,7 +140,7 @@ void MailThreadTest::testIndexInMixedOrder()
 
     {
         auto mail = Mail::create(mResourceInstanceIdentifier);
-        mail.setMimeMessage(message2->encodedContent());
+        mail.setMimeMessage(message2->encodedContent(true));
         mail.setFolder(folder);
         VERIFYEXEC(Store::create(mail));
     }
@@ -156,7 +156,7 @@ void MailThreadTest::testIndexInMixedOrder()
 
     {
         auto mail = Mail::create(mResourceInstanceIdentifier);
-        mail.setMimeMessage(message1->encodedContent());
+        mail.setMimeMessage(message1->encodedContent(true));
         mail.setFolder(folder);
         VERIFYEXEC(Store::create(mail));
     }
diff --git a/tests/messagequeuetest.cpp b/tests/messagequeuetest.cpp
index 83fa23f..2e3bd75 100644
--- a/tests/messagequeuetest.cpp
+++ b/tests/messagequeuetest.cpp
@@ -9,8 +9,6 @@
 #include "log.h"
 #include "test.h"
 
-SINK_DEBUG_AREA("messagequeuetest")
-
 /**
  * Test of the messagequeue implementation.
  */
diff --git a/tests/notificationtest.cpp b/tests/notificationtest.cpp
index a34d325..f0d957e 100644
--- a/tests/notificationtest.cpp
+++ b/tests/notificationtest.cpp
@@ -67,12 +67,21 @@ private slots:
         VERIFYEXEC(Sink::Store::synchronize(query));
         VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1"));
 
-        QVERIFY(statusNotifications.size() <= 3);
-        QTRY_COMPARE(statusNotifications.size(), 3);
-        //Sync
-        QCOMPARE(statusNotifications.at(0).code, static_cast<int>(ApplicationDomain::Status::ConnectedStatus));
-        QCOMPARE(statusNotifications.at(1).code, static_cast<int>(ApplicationDomain::Status::BusyStatus));
-        QCOMPARE(statusNotifications.at(2).code, static_cast<int>(ApplicationDomain::Status::ConnectedStatus));
+        using namespace Sink::ApplicationDomain;
+        {
+            QList<Status> expected = {
+                Status::ConnectedStatus,
+                Status::BusyStatus,
+                Status::ConnectedStatus,
+            };
+            qInfo() << "Received notifications " << statusNotifications;
+            QVERIFY2(statusNotifications.size() <= expected.size(), "More notifications than expected.");
+            QTRY_COMPARE(statusNotifications.size(), expected.size());
+            qInfo() << "All received notifications " << statusNotifications;
+            for (auto i = 0; i < statusNotifications.size(); i++) {
+                QCOMPARE(statusNotifications.at(i).code, static_cast<int>(expected.at(i)));
+            }
+        }
         //Changereplay
         // It can happen that we get a changereplay notification pair first and then a second one at the end,
         // we therefore currently filter all changereplay notifications (see above).
diff --git a/tests/querytest.cpp b/tests/querytest.cpp
index f639d94..714e549 100644
--- a/tests/querytest.cpp
+++ b/tests/querytest.cpp
@@ -98,10 +98,7 @@ private slots:
     void testSingle()
     {
         // Setup
-        {
-            Mail mail("sink.dummy.instance1");
-            VERIFYEXEC(Sink::Store::create<Mail>(mail));
-        }
+        VERIFYEXEC(Sink::Store::create<Mail>(Mail("sink.dummy.instance1")));
 
         // Test
         Sink::Query query;
@@ -116,10 +113,7 @@ private slots:
     void testSingleWithDelay()
     {
         // Setup
-        {
-            Mail mail("sink.dummy.instance1");
-            Sink::Store::create<Mail>(mail).exec().waitForFinished();
-        }
+        VERIFYEXEC(Sink::Store::create<Mail>(Mail("sink.dummy.instance1")));
 
         // Test
         Sink::Query query;
@@ -142,13 +136,13 @@ private slots:
             Mail mail("sink.dummy.instance1");
             mail.setExtractedMessageId("test1");
             mail.setFolder("folder1");
-            Sink::Store::create<Mail>(mail).exec().waitForFinished();
+            VERIFYEXEC(Sink::Store::create<Mail>(mail));
         }
         {
             Mail mail("sink.dummy.instance1");
             mail.setExtractedMessageId("test2");
             mail.setFolder("folder2");
-            Sink::Store::create<Mail>(mail).exec().waitForFinished();
+            VERIFYEXEC(Sink::Store::create<Mail>(mail));
         }
 
         // Test
@@ -164,13 +158,13 @@ private slots:
         auto mail = model->index(0, 0, QModelIndex()).data(Sink::Store::DomainObjectRole).value<Mail::Ptr>();
         {
             mail->setFolder("folder2");
-            Sink::Store::modify<Mail>(*mail).exec().waitForFinished();
+            VERIFYEXEC(Sink::Store::modify<Mail>(*mail));
         }
         QTRY_COMPARE(model->rowCount(), 0);
 
         {
             mail->setFolder("folder1");
-            Sink::Store::modify<Mail>(*mail).exec().waitForFinished();
+            VERIFYEXEC(Sink::Store::modify<Mail>(*mail));
         }
         QTRY_COMPARE(model->rowCount(), 1);
     }
@@ -181,8 +175,8 @@ private slots:
         // Setup
         {
             Mail mail("sink.dummy.instance1");
-            Sink::Store::create<Mail>(mail).exec().waitForFinished();
-            Sink::Store::create<Mail>(mail).exec().waitForFinished();
+            VERIFYEXEC(Sink::Store::create<Mail>(mail));
+            VERIFYEXEC(Sink::Store::create<Mail>(mail));
 
             Sink::Query query;
             query.resourceFilter("sink.dummy.instance1");
@@ -211,7 +205,7 @@ private slots:
         // Setup
         {
             Folder folder("sink.dummy.instance1");
-            Sink::Store::create<Folder>(folder).exec().waitForFinished();
+            VERIFYEXEC(Sink::Store::create<Folder>(folder));
         }
 
         // Test
@@ -387,6 +381,7 @@ private slots:
     {
         // Setup
         Folder::Ptr folderEntity;
+        const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0));
         {
             Folder folder("sink.dummy.instance1");
             Sink::Store::create<Folder>(folder).exec().waitForFinished();
@@ -404,7 +399,6 @@ private slots:
             folderEntity = model->index(0, 0).data(Sink::Store::DomainObjectRole).value<Folder::Ptr>();
             QVERIFY(!folderEntity->identifier().isEmpty());
 
-            const auto date = QDateTime(QDate(2015, 7, 7), QTime(12, 0));
             {
                 Mail mail("sink.dummy.instance1");
                 mail.setExtractedMessageId("testSecond");
@@ -434,6 +428,11 @@ private slots:
         query.filter<Mail::Folder>(*folderEntity);
         query.sort<Mail::Date>();
         query.limit(1);
+        query.setFlags(Query::LiveQuery);
+        query.reduce<ApplicationDomain::Mail::ThreadId>(Query::Reduce::Selector::max<ApplicationDomain::Mail::Date>())
+            .count("count")
+            .collect<ApplicationDomain::Mail::Unread>("unreadCollected")
+            .collect<ApplicationDomain::Mail::Important>("importantCollected");
 
         // Ensure all local data is processed
         VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << "sink.dummy.instance1"));
@@ -449,6 +448,26 @@ private slots:
         QCOMPARE(model->rowCount(), 2);
         // We can't make any assumptions about the order of the indexes
         // QCOMPARE(model->index(1, 0).data(Sink::Store::DomainObjectRole).value<Mail::Ptr>()->getProperty("messageId").toByteArray(), QByteArray("testSecond"));
+
+        //New revisions always go through
+        {
+            Mail mail("sink.dummy.instance1");
+            mail.setExtractedMessageId("testInjected");
+            mail.setFolder(folderEntity->identifier());
+            mail.setExtractedDate(date.addDays(-2));
+            Sink::Store::create<Mail>(mail).exec().waitForFinished();
+        }
+        QTRY_COMPARE(model->rowCount(), 3);
+
+        //Ensure we can continue fetching after the incremental update
+        model->fetchMore(QModelIndex());
+        QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool());
+        QCOMPARE(model->rowCount(), 4);
+
+        //Ensure we have fetched all
+        model->fetchMore(QModelIndex());
+        QTRY_VERIFY(model->data(QModelIndex(), Sink::Store::ChildrenFetchedRole).toBool());
+        QCOMPARE(model->rowCount(), 4);
     }
 
     void testReactToNewResource()
@@ -503,12 +522,13 @@ private slots:
             Folder folder2(resource2.identifier());
             VERIFYEXEC(Sink::Store::create<Folder>(folder2));
         }
+        VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource1.identifier()));
+        VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(QByteArrayList() << resource2.identifier()));
 
         // Test
         Sink::Query query;
         query.resourceFilter<SinkResource::Account>(account1);
 
-        // We fetch before the data is available and rely on the live query mechanism to deliver the actual data
         auto folders = Sink::Store::read<Folder>(query);
         QCOMPARE(folders.size(), 1);
     }
@@ -621,18 +641,50 @@ private slots:
         resource2.setResourceType("sink.dummy");
         VERIFYEXEC(Store::create(resource2));
 
-        Folder folder1(resource1.identifier());
-        VERIFYEXEC(Sink::Store::create<Folder>(folder1));
-        Folder folder2(resource2.identifier());
-        VERIFYEXEC(Sink::Store::create<Folder>(folder2));
+        VERIFYEXEC(Sink::Store::create<Folder>(Folder{resource1.identifier()}));
+        VERIFYEXEC(Sink::Store::create<Folder>(Folder{resource2.identifier()}));
 
-        // Test
-        Sink::Query query;
-        query.resourceContainsFilter<SinkResource::Capabilities>("cap1");
+        VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource1.identifier()));
+        VERIFYEXEC(Sink::ResourceControl::flushMessageQueue(resource2.identifier()));
 
         // We fetch before the data is available and rely on the live query mechanism to deliver the actual data
-        auto folders = Sink::Store::read<Folder>(query);
+        auto folders = Sink::Store::read<Folder>(Sink::Query{}.resourceContainsFilter<SinkResource::Capabilities>("cap1"));
         QCOMPARE(folders.size(), 1);
+
+        //TODO this should be part of the regular cleanup between tests
+        VERIFYEXEC(Store::remove(resource1));
+        VERIFYEXEC(Store::remove(resource2));
+    }
+
+    void testFilteredLiveResourceSubQuery()
+    {
+        using namespace Sink;
+        using namespace Sink::ApplicationDomain;
+
+        //Setup
+        auto resource1 = ApplicationDomainType::createEntity<SinkResource>();
+        resource1.setResourceType("sink.dummy");
+        resource1.setCapabilities(QByteArrayList() << "cap1");
+        VERIFYEXEC(Store::create(resource1));
+        VERIFYEXEC(Store::create<Folder>(Folder{resource1.identifier()}));
+        VERIFYEXEC(ResourceControl::flushMessageQueue(resource1.identifier()));
+
+        auto model = Sink::Store::loadModel<Folder>(Query{Query::LiveQuery}.resourceContainsFilter<SinkResource::Capabilities>("cap1"));
+        QTRY_COMPARE(model->rowCount(), 1);
+
+        auto resource2 = ApplicationDomainType::createEntity<SinkResource>();
+        resource2.setCapabilities(QByteArrayList() << "cap2");
+        resource2.setResourceType("sink.dummy");
+        VERIFYEXEC(Store::create(resource2));
+        VERIFYEXEC(Store::create<Folder>(Folder{resource2.identifier()}));
+        VERIFYEXEC(ResourceControl::flushMessageQueue(resource2.identifier()));
+
+        //The new resource should be filtered and thus not make it in here
+        QCOMPARE(model->rowCount(), 1);
+
+        //TODO this should be part of the regular cleanup between tests
+        VERIFYEXEC(Store::remove(resource1));
+        VERIFYEXEC(Store::remove(resource2));
     }
 
     void testLivequeryUnmatchInThread()
@@ -1059,30 +1111,18 @@ private slots:
         QDateTime now{QDate{2017, 2, 3}, QTime{10, 0, 0}};
         QDateTime later{QDate{2017, 2, 3}, QTime{11, 0, 0}};
 
-        {
-            auto mail1 = Mail::createEntity<Mail>("sink.dummy.instance1");
-            mail1.setExtractedMessageId("mail1");
-            mail1.setFolder(folder1);
-            mail1.setExtractedDate(now);
-            mail1.setImportant(false);
-            VERIFYEXEC(Sink::Store::create(mail1));
-        }
-        {
-            auto mail2 = Mail::createEntity<Mail>("sink.dummy.instance1");
-            mail2.setExtractedMessageId("mail2");
-            mail2.setFolder(folder1);
-            mail2.setExtractedDate(earlier);
-            mail2.setImportant(false);
-            VERIFYEXEC(Sink::Store::create(mail2));
-        }
-        {
-            auto mail3 = Mail::createEntity<Mail>("sink.dummy.instance1");
-            mail3.setExtractedMessageId("mail3");
-            mail3.setFolder(folder1);
-            mail3.setExtractedDate(later);
-            mail3.setImportant(true);
-            VERIFYEXEC(Sink::Store::create(mail3));
-        }
+        auto createMail = [] (const QByteArray &messageid, const Folder &folder, const QDateTime &date, bool important) {
+            auto mail = Mail::createEntity<Mail>("sink.dummy.instance1");
+            mail.setExtractedMessageId(messageid);
+            mail.setFolder(folder);
+            mail.setExtractedDate(date);
+            mail.setImportant(important);
+            return mail;
+        };
+
+        VERIFYEXEC(Sink::Store::create(createMail("mail1", folder1, now, false)));
+        VERIFYEXEC(Sink::Store::create(createMail("mail2", folder1, earlier, false)));
+        VERIFYEXEC(Sink::Store::create(createMail("mail3", folder1, later, true)));
 
         // Ensure all local data is processed
         VERIFYEXEC(Sink::ResourceControl::flushMessageQueue("sink.dummy.instance1"));
diff --git a/tests/storagetest.cpp b/tests/storagetest.cpp
index 7202628..9e9bad9 100644
--- a/tests/storagetest.cpp
+++ b/tests/storagetest.cpp
@@ -487,6 +487,61 @@ private slots:
         }
     }
 
+    void testCopyTransaction()
+    {
+        Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadWrite);
+        {
+            auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadWrite);
+            transaction.openDatabase("a", nullptr, false);
+            transaction.openDatabase("b", nullptr, false);
+            transaction.openDatabase("c", nullptr, false);
+            transaction.commit();
+        }
+        auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
+        for (int i = 0; i < 1000; i++) {
+            transaction.openDatabase("a", nullptr, false);
+            transaction.openDatabase("b", nullptr, false);
+            transaction.openDatabase("c", nullptr, false);
+            transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
+        }
+    }
+
+    /*
+     * This test is meant to find problems with the multi-process architecture and initial database creation.
+     * If we create named databases dynamically (not all up front), it is possilbe that we violate the rule
+     * that mdb_open_dbi may only be used by a single thread at a time.
+     * This test is meant to stress that condition.
+     *
+     * However, it yields absolutely nothing.
+     */
+    void testReadDuringExternalProcessWrite()
+    {
+        QSKIP("Not running multiprocess test");
+
+        QList<QFuture<void>> futures;
+        for (int i = 0; i < 5; i++) {
+            futures <<  QtConcurrent::run([&]() {
+                QTRY_VERIFY(Sink::Storage::DataStore(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly).exists());
+                Sink::Storage::DataStore store(testDataPath, dbName, Sink::Storage::DataStore::ReadOnly);
+                auto transaction = store.createTransaction(Sink::Storage::DataStore::ReadOnly);
+                for (int i = 0; i < 100000; i++) {
+                    transaction.openDatabase("a", nullptr, false);
+                    transaction.openDatabase("b", nullptr, false);
+                    transaction.openDatabase("c", nullptr, false);
+                    transaction.openDatabase("p", nullptr, false);
+                    transaction.openDatabase("q", nullptr, false);
+                }
+            });
+        }
+
+        //Start writing to the db from a separate process
+        QVERIFY(QProcess::startDetached(QCoreApplication::applicationDirPath() + "/dbwriter", QStringList() << testDataPath << dbName << QString::number(100000)));
+
+        for (auto future : futures) {
+            future.waitForFinished();
+        }
+
+    }
 };
 
 QTEST_MAIN(StorageTest)
diff --git a/tests/testimplementations.h b/tests/testimplementations.h
index a145265..ff9d9b8 100644
--- a/tests/testimplementations.h
+++ b/tests/testimplementations.h
@@ -33,7 +33,7 @@
 #include "mail_generated.h"
 #include "createentity_generated.h"
 
-class TestEventAdaptorFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Event, Sink::ApplicationDomain::Buffer::Event, Sink::ApplicationDomain::Buffer::EventBuilder>
+class TestEventAdaptorFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Event>
 {
 public:
     TestEventAdaptorFactory() : DomainTypeAdaptorFactory()
@@ -43,7 +43,7 @@ public:
     virtual ~TestEventAdaptorFactory(){};
 };
 
-class TestMailAdaptorFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Mail, Sink::ApplicationDomain::Buffer::Mail, Sink::ApplicationDomain::Buffer::MailBuilder>
+class TestMailAdaptorFactory : public DomainTypeAdaptorFactory<Sink::ApplicationDomain::Mail>
 {
 public:
     TestMailAdaptorFactory() : DomainTypeAdaptorFactory()